|
a/scripts/refresh-all-repos.py |
|
b/scripts/refresh-all-repos.py |
| 1 |
import logging
|
1 |
import logging
|
| 2 |
import optparse
|
|
|
| 3 |
|
2 |
|
| 4 |
from pylons import c
|
3 |
from pylons import c
|
| 5 |
from ming.orm import ThreadLocalORMSession
|
4 |
from ming.orm import ThreadLocalORMSession
|
| 6 |
|
5 |
|
| 7 |
from allura import model as M
|
6 |
from allura import model as M
|
| 8 |
from allura.lib import utils
|
7 |
from allura.lib.utils import chunked_find
|
| 9 |
|
8 |
|
| 10 |
log = logging.getLogger(__name__)
|
9 |
log = logging.getLogger(__name__)
|
| 11 |
|
10 |
|
| 12 |
PAGESIZE=1024
|
11 |
def main(options):
|
|
|
12 |
q_project = {}
|
|
|
13 |
if options.nbhd:
|
|
|
14 |
nbhd = M.Neighborhood.query.get(url_prefix=options.nbhd)
|
|
|
15 |
if not nbhd:
|
|
|
16 |
return "Invalid neighborhood url prefix."
|
|
|
17 |
q_project['neighborhood_id'] = nbhd._id
|
|
|
18 |
if options.project:
|
|
|
19 |
q_project['shortname'] = options.project
|
|
|
20 |
elif options.project_regex:
|
|
|
21 |
q_project['shortname'] = {'$regex': options.project_regex}
|
| 13 |
|
22 |
|
| 14 |
def main():
|
|
|
| 15 |
parser = optparse.OptionParser(usage="""%prog -- [options] [/p/ someproject/optional-subproj mount-point]\n\n
|
|
|
| 16 |
Specify a neighborhood url-prefix, project shortname, and mountpoint to run for just one repo. Omit that
|
|
|
| 17 |
to run for all repos.
|
|
|
| 18 |
""")
|
|
|
| 19 |
parser.add_option(
|
|
|
| 20 |
'--clean', action='store_true', dest='clean', default=False,
|
|
|
| 21 |
help='remove all RepoObjects before refresh')
|
|
|
| 22 |
parser.add_option(
|
|
|
| 23 |
'--all', action='store_true', dest='all', default=False,
|
|
|
| 24 |
help='refresh all commits (not just the ones that are new')
|
|
|
| 25 |
parser.add_option(
|
|
|
| 26 |
'--notify', action='store_true', dest='notify', default=False,
|
|
|
| 27 |
help='send email notifications of new commits')
|
|
|
| 28 |
options, args = parser.parse_args()
|
|
|
| 29 |
if args:
|
|
|
| 30 |
nbhd = M.Neighborhood.query.get(url_prefix=args[0])
|
|
|
| 31 |
shortname = args[1]
|
|
|
| 32 |
mount_point = args[2]
|
|
|
| 33 |
q_project = {'shortname': shortname, 'neighborhood_id': nbhd._id}
|
|
|
| 34 |
projects = {shortname:[mount_point]}
|
|
|
| 35 |
else:
|
|
|
| 36 |
projects = {}
|
|
|
| 37 |
q_project = {}
|
|
|
| 38 |
log.info('Refreshing repositories')
|
23 |
log.info('Refreshing repositories')
|
| 39 |
if options.clean:
|
24 |
if options.clean:
|
| 40 |
log.info('Removing all repository objects')
|
25 |
log.info('Removing all repository objects')
|
| 41 |
M.repo.CommitDoc.m.remove({})
|
26 |
M.repo.CommitDoc.m.remove({})
|
| 42 |
M.repo.TreeDoc.m.remove({})
|
27 |
M.repo.TreeDoc.m.remove({})
|
| 43 |
M.repo.TreesDoc.m.remove({})
|
28 |
M.repo.TreesDoc.m.remove({})
|
| 44 |
M.repo.DiffInfoDoc.m.remove({})
|
29 |
M.repo.DiffInfoDoc.m.remove({})
|
| 45 |
M.repo.CommitRunDoc.m.remove({})
|
30 |
M.repo.CommitRunDoc.m.remove({})
|
|
|
31 |
|
| 46 |
for chunk in utils.chunked_find(M.Project, q_project):
|
32 |
for chunk in chunked_find(M.Project, q_project):
|
| 47 |
for p in chunk:
|
33 |
for p in chunk:
|
|
|
34 |
log.info("Refreshing repos for project '%s'." % p.shortname)
|
|
|
35 |
if options.dry_run:
|
|
|
36 |
continue
|
| 48 |
c.project = p
|
37 |
c.project = p
|
| 49 |
if projects:
|
38 |
if options.mount_point:
|
| 50 |
mount_points = projects[p.shortname]
|
39 |
mount_points = [options.mount_point]
|
| 51 |
else:
|
40 |
else:
|
| 52 |
mount_points = [ ac.options.mount_point
|
41 |
mount_points = [ac.options.mount_point for ac in
|
| 53 |
for ac in M.AppConfig.query.find(dict(project_id=p._id)) ]
|
42 |
M.AppConfig.query.find(dict(project_id=p._id))]
|
| 54 |
for app in (p.app_instance(mp) for mp in mount_points):
|
43 |
for app in (p.app_instance(mp) for mp in mount_points):
|
| 55 |
c.app = app
|
44 |
c.app = app
|
| 56 |
if not hasattr(app, 'repo'): continue
|
45 |
if not hasattr(app, 'repo'): continue
|
| 57 |
try:
|
46 |
try:
|
| 58 |
c.app.repo._impl._setup_hooks()
|
47 |
c.app.repo._impl._setup_hooks()
|
|
... |
|
... |
| 67 |
except:
|
56 |
except:
|
| 68 |
log.exception('Error refreshing %r', c.app.repo)
|
57 |
log.exception('Error refreshing %r', c.app.repo)
|
| 69 |
ThreadLocalORMSession.flush_all()
|
58 |
ThreadLocalORMSession.flush_all()
|
| 70 |
ThreadLocalORMSession.close_all()
|
59 |
ThreadLocalORMSession.close_all()
|
| 71 |
|
60 |
|
|
|
61 |
def parse_options():
|
|
|
62 |
import argparse
|
|
|
63 |
parser = argparse.ArgumentParser(description='Scan repos on filesytem and '
|
|
|
64 |
'update repo metadata in MongoDB. Run for all repos (no args), '
|
|
|
65 |
'or restrict by neighborhood, project, or code tool mount point.')
|
|
|
66 |
parser.add_argument('--nbhd', action='store', default='', dest='nbhd',
|
|
|
67 |
help='Restrict update to a particular neighborhood, e.g. /p/.')
|
|
|
68 |
parser.add_argument('--project', action='store', default='', dest='project',
|
|
|
69 |
help='Restrict update to a particular project. To specify a '
|
|
|
70 |
'subproject, use a slash: project/subproject.')
|
|
|
71 |
parser.add_argument('--project-regex', action='store', default='',
|
|
|
72 |
dest='project_regex',
|
|
|
73 |
help='Restrict update to projects for which the shortname matches '
|
|
|
74 |
'the provided regex.')
|
|
|
75 |
parser.add_argument('--mount_point', default='', dest='mount_point',
|
|
|
76 |
help='Restrict update to repos at the given tool mount point. ')
|
|
|
77 |
parser.add_argument('--clean', action='store_true', dest='clean',
|
|
|
78 |
default=False, help='Remove all repo-related mongo docs before '
|
|
|
79 |
'refresh.')
|
|
|
80 |
parser.add_argument('--all', action='store_true', dest='all', default=False,
|
|
|
81 |
help='Refresh all commits (not just the ones that are new).')
|
|
|
82 |
parser.add_argument('--notify', action='store_true', dest='notify',
|
|
|
83 |
default=False, help='Send email notifications of new commits.')
|
|
|
84 |
parser.add_argument('--dry-run', action='store_true', dest='dry_run',
|
|
|
85 |
default=False, help='Log names of projects that would have their '
|
|
|
86 |
'repos refreshed, but do not perform the actual refresh.')
|
|
|
87 |
return parser.parse_args()
|
|
|
88 |
|
| 72 |
if __name__ == '__main__':
|
89 |
if __name__ == '__main__':
|
| 73 |
main()
|
90 |
import sys
|
|
|
91 |
sys.exit(main(parse_options()))
|