import bson
import datetime
import json
import logging
import multiprocessing
import re
import string
import sys
import colander as col
from ming.orm import session, ThreadLocalORMSession
from pylons import c, g
from tg import config
from allura import model as M
from allura.lib import helpers as h
log = logging.getLogger(__name__)
class TroveCategory():
def __init__(self, root_type=''):
self.root_type = root_type
def deserialize(self, node, cstruct):
if cstruct is col.null:
return col.null
cat = M.TroveCategory.query.get(fullpath=cstruct)
if not cat:
cat = M.TroveCategory.query.get(fullname=cstruct)
if not cat:
raise col.Invalid(node,
'"%s" is not a valid trove category.' % cstruct)
if not cat.fullpath.startswith(self.root_type):
raise col.Invalid(node,
'"%s" is not a valid "%s" trove category.' %
(cstruct, self.root_type))
return cat
class User():
def deserialize(self, node, cstruct):
if cstruct is col.null:
return col.null
user = M.User.by_username(cstruct)
if not user:
raise col.Invalid(node,
'Invalid username "%s".' % cstruct)
return user
class ProjectName(object):
def __init__(self, name, shortname):
self.name = name
self.shortname = shortname
class ProjectNameType():
def deserialize(self, node, cstruct):
if cstruct is col.null:
return col.null
name = cstruct
shortname = re.sub("[^A-Za-z0-9 ]", "", name).lower()
shortname = re.sub(" ", "-", shortname)
return ProjectName(name, shortname)
class ProjectShortnameType():
def deserialize(self, node, cstruct):
if cstruct is col.null:
return col.null
col.Length(min=3, max=15)(node, cstruct)
col.Regex(r'^[A-z][-A-z0-9]{2,}$',
msg='Project shortname must begin with a letter, can '
'contain letters, numbers, and dashes, and must be '
'3-15 characters in length.')(node, cstruct)
return cstruct.lower()
class Award():
def __init__(self, nbhd):
self.nbhd = nbhd
def deserialize(self, node, cstruct):
if cstruct is col.null:
return col.null
award = M.Award.query.find(dict(short=cstruct,
created_by_neighborhood_id=self.nbhd._id)).first()
if not award:
# try to look up the award by _id
award = M.Award.query.find(dict(_id=bson.ObjectId(cstruct),
created_by_neighborhood_id=self.nbhd._id)).first()
if not award:
raise col.Invalid(node,
'Invalid award "%s".' % cstruct)
return award
class TroveTopics(col.SequenceSchema):
trove_topics = col.SchemaNode(TroveCategory("Topic"))
class TroveLicenses(col.SequenceSchema):
trove_license = col.SchemaNode(TroveCategory("License"))
class TroveDatabases(col.SequenceSchema):
trove_databases = col.SchemaNode(TroveCategory("Database Environment"))
class TroveStatuses(col.SequenceSchema):
trove_statuses = col.SchemaNode(TroveCategory("Development Status"))
class TroveAudiences(col.SequenceSchema):
trove_audience = col.SchemaNode(TroveCategory("Intended Audience"))
class TroveOSes(col.SequenceSchema):
trove_oses = col.SchemaNode(TroveCategory("Operating System"))
class TroveLanguages(col.SequenceSchema):
trove_languages = col.SchemaNode(TroveCategory("Programming Language"))
class TroveTranslations(col.SequenceSchema):
trove_translations = col.SchemaNode(TroveCategory("Translations"))
class TroveUIs(col.SequenceSchema):
trove_uis = col.SchemaNode(TroveCategory("User Interface"))
class Labels(col.SequenceSchema):
label = col.SchemaNode(col.Str())
class Project(col.MappingSchema):
name = col.SchemaNode(ProjectNameType())
shortname = col.SchemaNode(ProjectShortnameType(), missing=None)
summary = col.SchemaNode(col.Str(), missing='')
description = col.SchemaNode(col.Str(), missing='')
admin = col.SchemaNode(User())
private = col.SchemaNode(col.Bool(), missing=False)
labels = Labels(missing=[])
external_homepage = col.SchemaNode(col.Str(), missing='')
trove_root_databases = TroveDatabases(missing=None)
trove_developmentstatuses = TroveStatuses(validator=col.Length(max=6), missing=None)
trove_audiences = TroveAudiences(validator=col.Length(max=6), missing=None)
trove_licenses = TroveLicenses(validator=col.Length(max=6), missing=None)
trove_oses = TroveOSes(missing=None)
trove_languages = TroveLanguages(validator=col.Length(max=6), missing=None)
trove_topics = TroveTopics(validator=col.Length(max=3), missing=None)
trove_natlanguages = TroveTranslations(missing=None)
trove_environments = TroveUIs(missing=None)
def valid_shortname(project):
if project.shortname:
# already validated in ProjectShortnameType validator
return True
elif 3 <= len(project.name.shortname) <= 15:
return True
else:
return 'Project shortname "%s" must be between 3 and 15 characters' \
% project.name.shortname
class Projects(col.SequenceSchema):
project = Project(validator=col.Function(valid_shortname))
class Object(object):
def __init__(self, d):
self.__dict__.update(d)
def trove_ids(orig, new_):
if new_ is None: return orig
return list(set(t._id for t in list(new_)))
def create_project(p, nbhd, user, options):
worker_name = multiprocessing.current_process().name
M.session.artifact_orm_session._get().skip_mod_date = True
shortname = p.shortname or p.name.shortname
project = M.Project.query.get(shortname=shortname,
neighborhood_id=nbhd._id)
project_template = nbhd.get_project_template()
if project and not (options.update and p.shortname):
log.warning('[%s] Skipping existing project "%s". To update an existing '
'project you must provide the project shortname and run '
'this script with --update.' % (worker_name, shortname))
return 0
if not project:
log.info('[%s] Creating project "%s".' % (worker_name, shortname))
try:
project = nbhd.register_project(shortname,
p.admin,
project_name=p.name.name,
private_project=p.private)
except Exception, e:
log.error('[%s] %s' % (worker_name, str(e)))
return 0
else:
log.info('[%s] Updating project "%s".' % (worker_name, shortname))
project.notifications_disabled = True
if options.ensure_tools and 'tools' in project_template:
for i, tool in enumerate(project_template['tools'].iterkeys()):
tool_config = project_template['tools'][tool]
if project.app_instance(tool_config['mount_point']):
continue
tool_options = tool_config.get('options', {})
for k, v in tool_options.iteritems():
if isinstance(v, basestring):
tool_options[k] = string.Template(v).safe_substitute(
project.root_project.__dict__.get('root_project', {}))
project.install_app(tool,
mount_label=tool_config['label'],
mount_point=tool_config['mount_point'],
**tool_options)
project.summary = p.summary
project.short_description = p.description
project.external_homepage = p.external_homepage
project.last_updated = datetime.datetime.utcnow()
# These properties may have been populated by nbhd template defaults in
# register_project(). Overwrite if we have data, otherwise keep defaults.
project.labels = p.labels or project.labels
project.trove_root_database = trove_ids(project.trove_root_database, p.trove_root_databases)
project.trove_developmentstatus = trove_ids(project.trove_developmentstatus, p.trove_developmentstatuses)
project.trove_audience = trove_ids(project.trove_audience, p.trove_audiences)
project.trove_license = trove_ids(project.trove_license, p.trove_licenses)
project.trove_os = trove_ids(project.trove_os, p.trove_oses)
project.trove_language = trove_ids(project.trove_language, p.trove_languages)
project.trove_topic = trove_ids(project.trove_topic, p.trove_topics)
project.trove_natlanguage = trove_ids(project.trove_natlanguage, p.trove_natlanguages)
project.trove_environment = trove_ids(project.trove_environment, p.trove_environments)
for a in p.awards:
M.AwardGrant(app_config_id=bson.ObjectId(),
tool_version=dict(neighborhood='0'), award_id=a._id,
granted_to_project_id=project._id,
granted_by_neighborhood_id=nbhd._id)
project.notifications_disabled = False
with h.push_config(c, project=project, user=user):
ThreadLocalORMSession.flush_all()
g.post_event('project_updated')
session(project).clear()
return 0
def create_projects(projects, nbhd, user, options):
for p in projects:
r = create_project(Object(p), nbhd, user, options)
if r != 0:
sys.exit(r)
def main(options):
log.addHandler(logging.StreamHandler(sys.stdout))
log.setLevel(getattr(logging, options.log_level.upper()))
log.debug(options)
nbhd = M.Neighborhood.query.get(name=options.neighborhood)
if not nbhd:
return 'Invalid neighborhood "%s".' % options.neighborhood
admin = M.User.query.get(username=config.get('sfx.api.siteadmin', 'sf-robot'))
data = json.load(open(options.file, 'r'))
project = Project()
project.add(col.SchemaNode(col.Sequence(),
col.SchemaNode(Award(nbhd)),
name='awards', missing=[]))
schema = col.SchemaNode(col.Sequence(), project, name='project')
projects = schema.deserialize(data)
log.debug(projects)
chunks = [projects[i::options.nprocs] for i in range(options.nprocs)]
jobs = []
for i in range(options.nprocs):
p = multiprocessing.Process(target=create_projects,
args=(chunks[i], nbhd, admin, options), name='worker-' + str(i+1))
jobs.append(p)
p.start()
for j in jobs:
j.join()
if j.exitcode <> 0: return j.exitcode
return 0
def parse_options():
import argparse
parser = argparse.ArgumentParser(
description='Import Allura project(s) from JSON file')
parser.add_argument('file', metavar='JSON_FILE', type=str,
help='Path to JSON file containing project data.')
parser.add_argument('neighborhood', metavar='NEIGHBORHOOD', type=str,
help='Destination Neighborhood shortname.')
parser.add_argument('--log', dest='log_level', default='INFO',
help='Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL).')
parser.add_argument('--update', dest='update', default=False,
action='store_true',
help='Update existing projects. Without this option, existing '
'projects will be skipped.')
parser.add_argument('--ensure-tools', dest='ensure_tools', default=False,
action='store_true',
help='Check nbhd project template for default tools, and install '
'them on the project(s) if not already installed.')
parser.add_argument('--nprocs', '-n', action='store', dest='nprocs', type=int,
help='Number of processes to divide the work among.',
default=multiprocessing.cpu_count())
return parser.parse_args()
if __name__ == '__main__':
sys.exit(main(parse_options()))