Open Source Projects Europe

--- a/Allura/allura/migrations.py
+++ b/scripts/migrations/007-update-acls.py
@@ -1,544 +1,205 @@
-from collections import defaultdict
+import logging
+from optparse import OptionParser
+from pprint import pformat
+
+import bson
 from pylons import c
-from flyway import Migration
-import ming
-from ming.orm import mapper, ORMSession, session, state
+from ming.base import Object
 
 from allura import model as M
-from allura.lib import plugin
-from allura.ext.project_home import model as PM
-from forgetracker import model as TM
-from forgewiki import model as WM
-from forgediscussion import model as DM
-from forgegit import model as GitM
-from forgehg import model as HgM
-from forgesvn import model as SVNM
-
-STATS_COLLECTION_SIZE=100000
-
-class MigrateProjectsAndFiles(Migration):
-    version = 14
-
-    def up(self):
-        self._up_projects()
-        db = self.session.db
-        for collection in db.collection_names():
-            if collection.endswith('.files'):
-                self._up_collection(db, collection)
-
-    def down(self):
-        # Nothing to do, really, as long as we don't update
-        # any metadata while upgraded
-        pass
-
-    def _up_projects(self):
-        projects = self.session.db.project
-        for p in projects.find():
-            if p.get('database_uri').startswith('mongo://'):
-                p['database_uri'] = p['database_uri'].replace('mongo://', 'mongodb://')
-                projects.save(p)
-
-    def _up_collection(self, db, collection_name):
-        collection = db[collection_name]
-        # First, create a 'root' collection, clearing it out as well
-        root_collection = db[collection_name.split('.')[0]]
-        root_collection.remove({})
-        for doc in collection.find():
-            newdoc = dict(doc)
-            newdoc.update(doc['metadata'])
-            newdoc.pop('metadata')
-            newdoc['file_id'] = doc['_id']
-            for aid_name in ('page_id', 'ticket_id'):
-                if aid_name in newdoc:
-                    newdoc['artifact_id'] = newdoc.pop(aid_name)
-            if 'post_id' in newdoc:
-                # post_id is stored along with artifact_id in posts
-                newdoc['artifact_id'] = newdoc['post_id']
-            root_collection.save(newdoc)
-
-class CreateStatsCollection(Migration):
-    version = 13
-
-    def up(self):
-        if self.session.db.name == 'allura':
-            self.session.db.create_collection(
-                M.Stats.__mongometa__.name,
-                capped=True,
-                size=STATS_COLLECTION_SIZE*10,
-                max=STATS_COLLECTION_SIZE)
-
-    def down(self):
-        if self.session.db.name == 'allura':
-            self.session.db.drop_collection(
-                M.Stats.__mongometa__.name)
-
-class DeleteFlashMailboxes(Migration):
-    version = 12
-
-    def up(self):
-        if self.session.db.name == 'allura':
-            self.ormsession.remove(
-                M.Mailbox,
-                {'type':'flash'})
-
-    def down(self):
-        raise NotImplementedError, 'ClearMailboxes.down'
-
-class ClearMailboxes(Migration):
-    version = 11
-
-    def up(self):
-        if self.session.db.name == 'allura':
-            self.ormsession.remove(M.Mailbox, {})
-            self.ormsession.ensure_indexes(M.Mailbox)
-
-    def down(self):
-        raise NotImplementedError, 'ClearMailboxes.down'
-
-class AddMountLabels(Migration):
-    version = 10
-
-    def up(self):
-        configs = self.ormsession.find(M.AppConfig).all()
-        for config in configs:
-            config.options['mount_label'] = config.options['mount_point']
-        self.ormsession.flush()
-
-    def down(self):
-        configs = self.ormsession.find(M.AppConfig).all()
-        for config in configs:
-            del config.options['mount_label']
-        self.ormsession.flush()
-
-class UpdateThemeToOnyx(Migration):
-    version = 9
-
-    def up(self):
-        if self.session.db.name == 'allura':
-            theme = self.ormsession.find(M.Theme, {'name':'forge_default'}).first()
-            if not theme: return
-            theme.color1='#295d78'
-            theme.color2='#272727'
-            theme.color3='#454545'
-            theme.color4='#c3c3c3'
-            theme.color5='#d7d7d7'
-            theme.color6='#ebebeb'
-            self.ormsession.update_now(theme, state(theme))
-            self.ormsession.flush()
-
-    def down(self):
-        if self.session.db.name == 'allura':
-            theme = self.ormsession.find(M.Theme, {'name':'forge_default'}).first()
-            if not theme: return
-            theme.color1='#0088cc'
-            theme.color2='#000000'
-            theme.color3='#454545'
-            theme.color4='#6c7681'
-            theme.color5='#d8d8d8'
-            theme.color6='#ececec'
-            self.ormsession.update_now(theme, state(theme))
-            self.ormsession.flush()
-
-class RemoveOldInitProjects(Migration):
-    version=8
-
-    def __init__(self, *args, **kwargs):
-        super(RemoveOldInitProjects, self).__init__(*args, **kwargs)
-        try:
-            c.project
-        except TypeError:
-            class EmptyClass(): pass
-            c._push_object(EmptyClass())
-            c.project = EmptyClass()
-            c.project._id = None
-            c.app = EmptyClass()
-            c.app.config = EmptyClass()
-            c.app.config.options = EmptyClass()
-            c.app.config.options.mount_point = None
-
-    def up(self):
-        self.ormsession.remove(M.Project, {'shortname':'--init--'})
-        self.ormsession.update(M.Project, {'shortname':'__init__'}, {'$set':{'shortname':'--init--'}})
-
-    def down(self):
-        pass # Do nothing
-
-class UnderToDash(Migration):
-    version = 7
-
-    def __init__(self, *args, **kwargs):
-        super(UnderToDash, self).__init__(*args, **kwargs)
-        try:
-            c.project
-        except TypeError:
-            class EmptyClass(): pass
-            c._push_object(EmptyClass())
-            c.project = EmptyClass()
-            c.project._id = None
-            c.app = EmptyClass()
-            c.app.config = EmptyClass()
-            c.app.config.options = EmptyClass()
-            c.app.config.options.mount_point = None
-
-    def up(self):
-        def fixup(s):
-            return s.replace('_', '-')
-        fix_pathnames(self.ormsession,fixup)
-
-    def down(self):
-        pass # Do nothing
-
-class MergeDuplicateRoles(Migration):
-    version = 6
-
-    def up(self):
-        if self.session.db.name == 'allura': self.up_allura()
-        else: self.up_project()
-
-    def down(self):
-        pass
-
-    def up_allura(self):
-        # Uniquify User.projects list
-        for u in self.session.find(self.User):
-            u.projects = list(set(u.projects))
-            self.session.save(u)
-
-    def up_project(self):
-        # Consolidate roles by user_id
-        roles_by_user = defaultdict(list)
-        for role in self.session.find(self.Role):
-            if role.get('user_id') is None: continue
-            roles_by_user[role.user_id].append(role)
-        for user_id, roles in roles_by_user.iteritems():
-            if len(roles) <= 1: continue
-            main_role = roles[0]
-            subroles = set()
-            for r in roles:
-                for sr_id in r.get('roles', []):
-                    subroles.add(sr_id)
-            main_role.roles = list(subroles)
-            self.session.save(main_role)
-            for r in roles[1:]:
-                self.session.delete(r)
-        # Add index
-        try:
-            self.session.drop_indexes(self.Role)
-        except:
-            pass
-        self.session.ensure_indexes(self.Role)
-
-    class User(ming.Document):
-        class __mongometa__:
-            name='user'
-
-    class Role(ming.Document):
-        class __mongometa__:
-            name='user'
-            unique_indexes = [ ('user_id', 'name') ]
-
-class UnifyPermissions(Migration):
-    version = 5
-
-    def up(self):
-        perm_owner = self.ormsession.find(M.ProjectRole, dict(name='owner')).first()
-        perm_Owner = self.ormsession.find(M.ProjectRole, dict(name='Owner')).first()
-        if perm_owner:
-            perm_owner.name = 'Admin'
-        elif perm_Owner:
-            perm_Owner.name = 'Admin'
-        perm_member = self.ormsession.find(M.ProjectRole, dict(name='member')).first()
-        if perm_member:
-            perm_member.name = 'Member'
-        perm_developer = self.ormsession.find(M.ProjectRole, dict(name='developer')).first()
-        if perm_developer:
-            perm_developer.name = 'Developer'
-        if self.session.db.name != 'allura':
-            role_names = [r.name for r in self.ormsession.find(M.ProjectRole)]
-            if len(role_names) and 'Admin' not in role_names:
-                new_admin = M.ProjectRole(name='Admin')
-            if len(role_names) and 'Developer' not in role_names:
-                new_admin = M.ProjectRole(name='Developer')
-
-        self.ormsession.flush()
-
-    def down(self):
-        perm_Admin = self.ormsession.find(M.ProjectRole, dict(name='Admin')).first()
-        if perm_Admin:
-            perm_Admin.name = 'Owner'
-
-        self.ormsession.flush()
-
-class UpdateProjectsToTools(Migration):
-    version = 4
-
-    def up(self):
-        # if self.session.db.name == 'allura':
-        # import pdb; pdb.set_trace()
-        rename_key(self.session,M.User,'plugin_preferences','tool_preferences')
-        rename_key(self.session,M.AppConfig,'plugin_name','tool_name')
-        rename_key(self.session,M.ArtifactLink,'plugin_name','tool_name')
-        rename_key(self.session,M.Project,'plugin','tool',inside='acl')
-        if c.app:
-            c.app.__version__ = '0.1'
-            c.app.config.tool_name = 'project_home'
-        for pc in self.ormsession.find(PM.PortalConfig, {}):
-            for div in pc.layout:
-                for widget in div.content:
-                    if widget.widget_name == 'plugin_status':
-                        widget.widget_name = 'tool_status'
-        self.ormsession.flush()
-        # fix artifacts
-        for cls in (
-            M.Artifact,
-            M.VersionedArtifact,
-            M.Snapshot,
-            M.Message,
-            M.Post,
-            M.AwardGrant,
-            M.Discussion,
-            M.Award,
-            M.Thread,
-            M.Post,
-            M.PostHistory,
-            DM.Forum,
-            DM.ForumPost,
-            DM.forum.ForumPostHistory,
-            DM.ForumThread,
-            WM.Page,
-            WM.wiki.PageHistory,
-            TM.Bin,
-            TM.Ticket,
-            TM.ticket.TicketHistory,
-            PM.PortalConfig,
-            GitM.GitRepository,
-            HgM.HgRepository,
-            SVNM.SVNRepository,
-            ):
-            rename_key(self.session,cls,'plugin_verson','tool_version')
-
-    def down(self):
-        # if self.session.db.name == 'allura':
-        rename_key(self.session,M.User,'tool_preferences','plugin_preferences')
-        rename_key(self.session,M.AppConfig,'tool_name','plugin_name')
-        rename_key(self.session,M.ArtifactLink,'tool_name','plugin_name')
-        rename_key(self.session,M.Project,'tool','plugin',inside='acl')
-        if c.app:
-            c.app.__version__ = '0.1'
-            c.app.config.tool_name = 'project_home'
-        for pc in self.ormsession.find(PM.PortalConfig, {}):
-            for div in pc.layout:
-                for widget in div.content:
-                    if widget.widget_name == 'tool_status':
-                        widget.widget_name = 'plugin_status'
-        self.ormsession.flush()
-        # fix artifacts
-        for cls in (
-            M.Artifact,
-            M.VersionedArtifact,
-            M.Snapshot,
-            M.Message,
-            M.Post,
-            M.AwardGrant,
-            M.Discussion,
-            M.Award,
-            M.Thread,
-            M.Post,
-            M.PostHistory,
-            DM.Forum,
-            DM.ForumPost,
-            DM.forum.ForumPostHistory,
-            DM.ForumThread,
-            WM.Page,
-            WM.wiki.PageHistory,
-            GitM.GitRepository,
-            HgM.HgRepository,
-            SVNM.SVNRepository,
-            TM.Bin,
-            TM.Ticket,
-            TM.ticket.TicketHistory,
-            PM.PortalConfig,
-            ):
-            rename_key(self.session,cls,'tool_version','plugin_verson')
-
-class UpdateThemeToShinyBook(Migration):
-    version = 3
-
-    def up(self):
-        if self.session.db.name == 'allura':
-            theme = self.ormsession.find(M.Theme, {'name':'forge_default'}).first()
-            if not theme: return
-            theme.color1='#0088cc'
-            theme.color2='#000000'
-            theme.color3='#454545'
-            theme.color4='#6c7681'
-            theme.color5='#d8d8d8'
-            theme.color6='#ececec'
-            self.ormsession.update_now(theme, state(theme))
-            self.ormsession.flush()
-
-    def down(self):
-        if self.session.db.name == 'allura':
-            theme = self.ormsession.find(M.Theme, {'name':'forge_default'}).first()
-            if not theme: return
-            theme.color1='#104a75'
-            theme.color2='#aed0ea'
-            theme.color3='#EDF3FB'
-            theme.color4='#D7E8F5'
-            theme.color5='#000'
-            theme.color6='#000'
-            self.ormsession.update_now(theme, state(theme))
-            self.ormsession.flush()
-
-class RenameNeighborhoods(Migration):
-    version = 2
-
-    def up(self):
-        n_users = self.ormsession.find(M.Neighborhood, dict(name='Users')).first()
-        n_projects = self.ormsession.find(M.Neighborhood, dict(name='Projects')).first()
-        if n_users:
-            n_users.url_prefix = '/u/'
-            n_users.shortname_prefix = 'u/'
-        if n_projects:
-            n_projects.url_prefix = '/p/'
-        for p in self.ormsession.find(M.Project, {}):
-            if p.shortname.startswith('users/'):
-                p.shortname = p.shortname.replace('users/', 'u/')
-        self.ormsession.flush()
-
-    def down(self):
-        n_users = self.ormsession.find(M.Neighborhood, dict(name='Users')).first()
-        n_projects = self.ormsession.find(M.Neighborhood, dict(name='Projects')).first()
-        if n_users:
-            n_users.url_prefix = '/users/'
-            n_users.shortname_prefix = 'users/'
-        if n_projects:
-            n_projects.url_prefix = '/projects/'
-        for p in self.ormsession.find(M.Project, {}):
-            if p.shortname.startswith('u/'):
-                p.shortname = p.shortname.replace('u/', 'users/')
-        self.ormsession.flush()
-
-class DowncaseMountPoints(Migration):
-    version = 1
-
-    def __init__(self, *args, **kwargs):
-        super(DowncaseMountPoints, self).__init__(*args, **kwargs)
-        try:
-            c.project
-        except TypeError:
-            class EmptyClass(): pass
-            c._push_object(EmptyClass())
-            c.project = EmptyClass()
-            c.project._id = None
-            c.app = EmptyClass()
-            c.app.config = EmptyClass()
-            c.app.config.options = EmptyClass()
-            c.app.config.options.mount_point = None
-
-    def up_requires(self):
-        yield ('ForgeWiki', 0)
-        yield ('ForgeTracker', 0)
-        yield ('pyforge', 0)
-
-    def up(self):
-        fix_pathnames(self.ormsession, lambda s:s.lower().replace(' ', '_'))
-
-    def down(self):
-        pass # Do nothing
-
-class V0(Migration):
-    version = 0
-    def up(self): pass
-    def down(self):  pass
-
-def rename_key(session, cls, old_key, new_key, inside=None):
-    pm = session._impl(mapper(cls).doc_cls)
-    for item in pm.find():
-        if inside:
-            if inside in item and old_key in item[inside]:
-                item[inside][new_key] = item[inside][old_key]
-                del item[inside][old_key]
-        else:
-            if old_key in item:
-                item[new_key] = item[old_key]
-                del item[old_key]
-        pm.save(item)
-
-def fix_pathnames(ormsession, mutator):
-    def fix_aref(aref):
-        if aref and aref.mount_point:
-            aref.mount_point = mutator(aref.mount_point)
-     # Fix neigborhoods
-    for n in ormsession.find(M.Neighborhood, {}):
-        n.shortname_prefix = mutator(n.shortname_prefix)
-   # Fix Projects
-    for p in ormsession.find(M.Project, {}):
-        p.shortname = mutator(p.shortname)
-    # Fix AppConfigs
-    for ac in ormsession.find(M.AppConfig, {}):
-        ac.options.mount_point = mutator(ac.options.mount_point)
-        if ac.tool_name == 'Forum':
-            ac.tool_name = 'Discussion'
-    ormsession.flush(); ormsession.clear()
-    # Fix ArtifactLinks
-    for al in ormsession.find(M.ArtifactLink, {}):
-        fix_aref(al.artifact_reference)
-    # Fix feeds
-    for f in ormsession.find(M.Feed, {}):
-        fix_aref(f.artifact_reference)
-    # Fix notifications
-    for n in ormsession.find(M.Notification, {}):
-        fix_aref(n.artifact_reference)
-    # Fix tags
-    for n in ormsession.find(M.TagEvent, {}):
-        fix_aref(n.artifact_ref)
-    for n in ormsession.find(M.UserTags, {}):
-        fix_aref(n.artifact_reference)
-    for n in ormsession.find(M.Tag, {}):
-        fix_aref(n.artifact_ref)
-    # fix PortalConfig
-    for pc in ormsession.find(PM.PortalConfig):
-        for layout in pc.layout:
-            for w in layout.content:
-                w.mount_point = mutator(w.mount_point)
-    # Fix thread (has explicit artifact_reference property)
-    for t in ormsession.find(M.Thread, {}):
-        fix_aref(t.artifact_reference)
-    for t in ormsession.find(DM.ForumThread, {}):
-        fix_aref(t.artifact_reference)
-    ormsession.flush(); ormsession.clear()
-    # fix artifacts
-    for cls in (
-        M.Artifact,
-        M.VersionedArtifact,
-        M.Snapshot,
-        M.Message,
-        M.Post,
-        M.AwardGrant,
-        M.Discussion,
-        M.Award,
-        M.Thread,
-        M.Post,
-        M.PostHistory,
-        DM.Forum,
-        DM.ForumPost,
-        DM.forum.ForumPostHistory,
-        DM.ForumThread,
-        WM.Page,
-        WM.wiki.PageHistory,
-        GitM.GitRepository,
-        HgM.HgRepository,
-        SVNM.SVNRepository,
-        TM.Bin,
-        TM.Ticket,
-        TM.ticket.TicketHistory,
-        PM.PortalConfig,
-        ):
-        for obj in ormsession.find(cls, {}):
-            for ref in obj.references:
-                fix_aref(ref)
-            for ref in obj.backreferences.itervalues():
-                fix_aref(ref)
-            ormsession.flush(); ormsession.clear()
-
+from allura.command.show_models import dfs, build_model_inheritance_graph
+
+log = logging.getLogger('update-acls')
+
+options = None
+optparser = OptionParser(usage='allurapaste script <ini file> -- %prog [options] [neighborhood1...]')
+optparser.add_option('-t', '--test',  dest='test', action='store_true')
+
+main_db = M.main_doc_session.db
+c_neighborhood =  main_db.neighborhood
+c_project =  main_db.project
+c_user = main_db.user
+c_project_role = main_db.project_role
+c.project = Object(
+    database_uri=c_project.find().next()['database_uri'])
+
+project_db = M.project_doc_session.db
+c_app_config = project_db.config
+
+def main():
+    global options
+    options, neighborhoods = optparser.parse_args()
+    if neighborhoods:
+        log.info('Updating neighborhoods: %s', neighborhoods)
+        q_neighborhoods = list(c_neighborhood.find(dict(name={'$in': neighborhoods })))
+        neighborhood_ids=[ n['_id'] for n in q_neighborhoods ]
+        q_projects = list(c_project.find(dict(neighborhood_id={'$in': neighborhood_ids})))
+        project_ids = list(p['_id'] for p in q_projects)
+        q_app_config = list(c_app_config.find(dict(project_id={'$in': project_ids})))
+        log.info('... %d neighborhoods', len(q_neighborhoods))
+        log.info('... %d projects', len(q_projects))
+        log.info('... %d app configs', len(q_app_config))
+    else:
+        q_neighborhoods = c_neighborhood.find()
+        q_projects = c_project.find()
+        q_app_config = c_app_config.find()
+        log.info('Updating all neighborhoods')
+    # Update project acls
+    log.info('====================================')
+    log.info('Update project ACLs')
+    for p in q_projects:
+        update_project_acl(p)
+        if not options.test: c_project.save(p)
+    # Update neighborhood acls
+    log.info('====================================')
+    log.info('Update neighborhood ACLs')
+    for n in q_neighborhoods:
+        p = c_project.find(dict(
+                neighborhood_id=n['_id'], shortname='--init--')).next()
+        update_neighborhood_acl(n,p)
+        if not options.test:
+            c_neighborhood.save(n)
+            c_project.save(p)
+    graph = build_model_inheritance_graph()
+    # Update app config acls
+    log.info('====================================')
+    log.info('Update appconfig ACLs')
+    for ac in q_app_config:
+        simple_acl_update(ac, 'app_config')
+        if not options.test: c_app_config.save(ac)
+        # Update artifact acls
+        log.info('====================================')
+        log.info('Update artifact ACLs for %s', ac['_id'])
+        for _, a_cls in dfs(M.Artifact, graph):
+            c_artifact = project_db[a_cls.__mongometa__.name]
+            for a in c_artifact.find(dict(app_config_id=ac['_id'])):
+                empty_acl = not a['acl']
+                simple_acl_update(a, a_cls.__mongometa__.name)
+                if not options.test and not empty_acl: c_artifact.save(a)
+
+def update_project_acl(project_doc):
+    '''Convert the old dict-style ACL to a list of ALLOW ACEs. Also move the
+    security,tool,delete perms to 'admin'
+    '''
+    if not isinstance(project_doc['acl'], dict):
+        log.warning('Project %s is already updated', project_doc['shortname'])
+        return
+    perm_map = dict(
+        read='read',
+        create='create',
+        update='update',
+        security='admin',
+        tool='admin',
+        delete='admin')
+    new_acl = []
+    for perm, role_ids in sorted(project_doc['acl'].iteritems()):
+        perm = perm_map[perm]
+        for rid in role_ids:
+            if c_project_role.find(dict(_id=rid)).count() == 0: continue
+            _grant(new_acl, perm, rid)
+    if options.test:
+        log.info('--- update %s\n%s\n%s\n---',
+                 project_doc['shortname'],
+                 pformat(_format_acd(project_doc['acl'])),
+                 pformat(map(_format_ace, new_acl)))
+    project_doc['acl'] = new_acl
+
+def update_neighborhood_acl(neighborhood_doc, init_doc):
+    '''Convert nbhd admins users to --init-- project admins'''
+    if options.test: log.info('Update nbhd %s', neighborhood_doc['name'])
+    if 'acl' not in neighborhood_doc:
+        log.warning('Neighborhood %s already updated', neighborhood_doc['name'])
+        return
+    p = Object(init_doc)
+    p.root_project=p
+    r_anon = _project_role(init_doc['_id'], '*anonymous')
+    r_auth = _project_role(init_doc['_id'], '*authenticated')
+    r_admin = _project_role(init_doc['_id'], 'Admin')
+    acl = neighborhood_doc['acl']
+    new_acl = list(init_doc['acl'])
+    assert acl['read'] == [None] # nbhd should be public
+    for uid in acl['admin'] + acl['moderate']:
+        u = c_user.find(dict(_id=uid)).next()
+        if options.test:
+            log.info('... grant nbhd admin to: %s', u['username'])
+            continue
+        role =  _project_role(init_doc['_id'], user_id=uid)
+        if r_admin['_id'] not in role['roles']:
+            role['roles'].append(r_admin['_id'])
+            c_project_role.save(role)
+    _grant(new_acl, 'read', r_anon['_id'])
+    _grant(new_acl, 'admin', r_admin['_id'])
+    _grant(new_acl, 'register', r_admin['_id'])
+    if acl['create'] == [ ]:
+        if options.test: log.info('grant register to auth')
+        _grant(new_acl, 'register', r_auth['_id'])
+    del neighborhood_doc['acl']
+    if options.test:
+        log.info('--- new init acl:\n%s\n%s\n---',
+                 pformat(_format_acd(init_doc['acl'])),
+                 pformat(map(_format_ace, new_acl)))
+    init_doc['acl'] = new_acl
+
+def _project_role(project_id, name=None, user_id=None):
+    doc = dict(project_id=project_id)
+    if name:
+        doc['name'] = name
+    else:
+        doc['user_id'] = user_id
+    for role in c_project_role.find(doc):
+        return role
+    assert name is None
+    doc.update(
+        _id=bson.ObjectId(),
+        roles=[])
+    c_project_role.save(doc)
+    return doc
+                                 
+
+def simple_acl_update(doc, collection_name):
+    '''Update dict-style to list-style ACL'''
+    if not isinstance(doc['acl'], dict):
+        log.warning('Already upgraded %s: %s', collection_name, doc)
+        return
+
+    new_acl = []
+    for perm, role_ids in sorted(doc['acl'].iteritems()):
+        for rid in role_ids:
+            _grant(new_acl, perm, rid)
+    if options.test and doc['acl']:
+        log.info('--- update %s %s\n%s\n%s\n---',
+                 collection_name, doc['_id'],
+                 pformat(_format_acd(doc['acl'])),
+                 pformat(map(_format_ace, new_acl)))
+    doc['acl'] = new_acl
+
+def _grant(acl, permission, role_id):
+    ace = dict(
+        access='ALLOW',
+        permission=permission,
+        role_id=role_id)
+    if ace not in acl:
+        acl.append(ace)
+
+def _format_ace(ace):
+    if isinstance(ace, basestring): return ace
+    return '(%s, %s, %s)' % (
+        ace['access'], ace['permission'], _format_role(ace['role_id']))
+
+def _format_role(rid):
+    for role in c_project_role.find(dict(_id=rid)):
+        if role['name']:
+            return role['name']
+        if role['user_id']:
+            u = c_user.find(_id=role['user_id']).next()
+            return u['username']
+        break
+    return '--invalid--'
+
+def _format_acd(acd):
+    return dict(
+        (k, map(_format_role, v))
+        for k,v in acd.iteritems())
+
+if __name__ == '__main__':
+    main()

--- a/ForgeTracker/forgetracker/model/migrations.py
+++ b/scripts/migrations/012-uninstall-home.py
@@ -1,168 +1,121 @@
-import cPickle as pickle
-from itertools import chain
+import sys
+import logging
 
-import bson
-from ming.orm import state
 from pylons import c
+from ming.orm import session
+from bson import ObjectId
+from mock import Mock, patch
 
-from flyway import Migration
-from allura.model import Thread, AppConfig, ArtifactReference, ProjectRole
-from forgetracker.model import Ticket, Globals
+from allura.lib import helpers as h
+from allura import model as M
+from forgewiki import model as WM
+from allura.ext.project_home import ProjectHomeApp
 
+log = logging.getLogger('uninstall-home')
+log.addHandler(logging.StreamHandler(sys.stdout))
 
-class TrackerMigration(Migration):
-    def __init__(self, *args, **kwargs):
-        super(TrackerMigration, self).__init__(*args, **kwargs)
-        try:
-            c.project
-        except (TypeError, AttributeError), exc:
-            class EmptyClass(): pass
-            c._push_object(EmptyClass())
-            c.project = EmptyClass()
-            c.project._id = None
-            c.app = EmptyClass()
-            c.app.config = EmptyClass()
-            c.app.config.options = EmptyClass()
-            c.app.config.options.mount_point = None
+def main():
+    test = sys.argv[-1] == 'test'
+    log.info('Removing "home" tools')
+    affected_projects = 0
+    possibly_orphaned_projects = 0
+    solr_delete = Mock()
+    notification_post = Mock()
+    for some_projects in chunked_project_iterator({'neighborhood_id': {'$ne': ObjectId("4be2faf8898e33156f00003e")}}):
+        for project in some_projects:
+            c.project = project
+            old_home_app = project.app_instance('home')
+            if isinstance(old_home_app, ProjectHomeApp):
 
+                # would we actually be able to install a wiki?
+                if M.ProjectRole.by_name('Admin') is None:
+                    log.warning('project %s may be orphaned' % project.shortname)
+                    possibly_orphaned_projects += 1
+                    continue
 
-class V0(TrackerMigration):
-    '''Migrate Thread.artifact_id to Thread.artifact_reference'''
-    version = 0
+                affected_projects += 1
 
-    def up(self):
-        for pg in self.ormsession.find(Ticket):
-            q1 = self.ormsession.find(Thread, dict(artifact_id=pg._id))
-            q2 = self.ormsession.find(Thread, {'artifact_reference.artifact_id':pg._id})
-            for t in chain(q1, q2):
-                t.artifact_reference = self._dump_ref(pg)
-                t.artifact_id = None
-                self.ormsession.update_now(t, state(t))
-        self.ormsession.flush()
+                # remove the existing home tool
+                if test:
+                    log.info('would remove "home" tool from project ' + project.shortname)
+                else:
+                    log.info('removing "home" tool from project ' + project.shortname)
+                    with patch('allura.app.g.solr.delete', solr_delete):
+                        project.uninstall_app('home')
 
-    def down(self):
-        for pg in self.ormsession.find(Ticket):
-            for t in self.ormsession.find(Thread, dict(artifact_reference=self._dump_ref(pg))):
-                t.artifact_id = pg._id
-                t.artifact_reference = None
-                self.ormsession.update_now(t, state(t))
-        self.ormsession.flush()
+                # ...and put a Wiki in its place (note we only create a Wiki if we deleted the old home)
+                if test:
+                    log.info('would create Wiki "home" for project ' + project.shortname)
+                else:
+                    log.info('creating Wiki "home" for project ' + project.shortname)
+                    home_title = project.homepage_title or 'Home'
+                    home_text = project.description or ''
+                    if home_text == 'You can edit this description in the admin page':
+                        home_text = 'You can edit this description'
 
+                    # re-number all the mounts so the new Wiki comes first
+                    mounts = project.ordered_mounts()
+                    with patch('forgewiki.model.wiki.Notification.post', notification_post):
+                        new_home_app = project.install_app('Wiki', 'home', 'Home')
+                    mounts = [{'ordinal':0, 'ac':new_home_app.config}] + mounts
+                    for i, mount in enumerate(mounts):
+                        if 'ac' in mount:
+                            mount['ac'].options['ordinal'] = i
+                            session(mount['ac']).flush()
+                        elif 'sub' in mount:
+                            mount['sub'].ordinal = i
+                            session(mount['sub']).flush()
 
-    def _dump_ref(self, art):
-        app_config = self.ormsession.get(AppConfig, art.app_config_id)
-        return ArtifactReference(dict(
-            project_id=app_config.project_id,
-            mount_point=app_config.options.mount_point,
-            artifact_type=bson.Binary(pickle.dumps(art.__class__)),
-            artifact_id=art._id))
+                    # make it look as much like the old home tool as possible
+                    new_home_app.config.options['show_left_bar'] = False
+                    new_home_app.config.options['show_discussion'] = False
 
+                    # now let's fix the home page itself
+                    log.info('updating home page to "%s"' % home_title)
+                    new_home_page = WM.Page.query.find(dict(app_config_id=new_home_app.config._id)).first()
+                    with h.push_config(c, app=new_home_app):
+                        if new_home_page is None:
+                            # weird: we didn't find the existing home page
+                            log.warning('hmmm, actually creating the home page ("%s") for project "%s" from scratch' % (home_title, project.shortname))
+                            new_home_page = WM.Page.upsert(home_title)
+                            new_home_page.viewable_by = ['all']
+                        new_home_page.title = home_title
+                        new_home_page.text = home_text
+                        with patch('forgewiki.model.wiki.Notification.post', notification_post):
+                            new_home_page.commit()
+                    assert new_home_page is not None
+                    assert new_home_page.title == home_title
+                    assert new_home_page.version == 2
 
-class AddShowInSearchAttributeToAllCustomFields(TrackerMigration):
-    version = 1
+                    # if we changed the home page name, make sure the Wiki knows that's the root page
+                    new_home_app.root_page_name = home_title
 
-    def up(self):
-        for custom_field in self.each_custom_field():
-            custom_field['show_in_search'] = False
-        self.ormsession.flush()
+                session(project).flush()
+            session(project).clear()
+    if test:
+        log.info('%s projects would be updated' % affected_projects)
+    else:
+        log.info('%s projects were updated' % affected_projects)
+    if possibly_orphaned_projects:
+        log.warning('%s possibly orphaned projects found' % possibly_orphaned_projects)
+    if not test:
+        assert solr_delete.call_count == affected_projects, solr_delete.call_count
+        assert notification_post.call_count == 2 * affected_projects, notification_post.call_count
 
-    def down(self):
-        for custom_field in self.each_custom_field():
-            del custom_field['show_in_search']
-        self.ormsession.flush()
+PAGESIZE=1024
 
-    def each_custom_field(self):
-        for tracker_globals in self.ormsession.find(Globals):
-            for custom_field in tracker_globals.custom_fields:
-                yield custom_field
+def chunked_project_iterator(q_project):
+    '''shamelessly copied from refresh-all-repos.py'''
+    page = 0
+    while True:
+        results = (M.Project.query
+                   .find(q_project)
+                   .skip(PAGESIZE*page)
+                   .limit(PAGESIZE)
+                   .all())
+        if not results: break
+        yield results
+        page += 1
 
-
-class AddSaveSearchesPermission(TrackerMigration):
-    version = 2
-
-    def up(self):
-        first_ticket = self.ormsession.find(Ticket).first()
-        if first_ticket:
-            app_config = self.ormsession.get(AppConfig, first_ticket.app_config_id)
-            app_config.acl.save_searches = []
-            developer = self.ormsession.find(ProjectRole, dict(name='Developer')).first()
-            admin = self.ormsession.find(ProjectRole, dict(name='Admin')).first()
-            if developer:
-                app_config.acl.save_searches.append(developer._id)
-            if admin:
-                app_config.acl.save_searches.append(admin._id)
-            self.ormsession.flush()
-
-    def down(self):
-        first_ticket = self.ormsession.find(Ticket).first()
-        if first_ticket:
-            app_config = self.ormsession.get(AppConfig, first_ticket.app_config_id)
-            del app_config.acl.save_searches
-            self.ormsession.flush()
-
-
-class SplitStatusNamesIntoOpenAndClosed(TrackerMigration):
-    version = 3
-
-    def up(self):
-        for tracker_globals in self.ormsession.find(Globals):
-            old_names = tracker_globals.status_names
-            tracker_globals.open_status_names = ' '.join([name for name in old_names.split(' ') if name and name != 'closed'])
-            tracker_globals.closed_status_names = 'closed'
-            tracker_globals.status_names = ''
-        self.ormsession.flush()
-        self.ormsession.clear()
-
-    def down(self):
-        for tracker_globals in self.ormsession.find(Globals):
-            tracker_globals.status_names = ' '.join([tracker_globals.open_status_names, tracker_globals.closed_status_names])
-            tracker_globals.open_status_names = ''
-            tracker_globals.closed_status_names = ''
-        self.ormsession.flush()
-        self.ormsession.clear()
-
-class MoveMilestonesToCustom(TrackerMigration):
-    version = 4
-
-    def _custom_field(self, tracker_globals):
-        names = tracker_globals.milestone_names or ''
-        return dict(
-            name='_milestone',
-            show_in_search=True,
-            type='milestone',
-            label='Milestone',
-            milestones=[
-                dict(name=name, complete=False, due_date=None)
-                for name in names.split() ])
-
-    def up(self):
-        for tracker_globals in self.ormsession.find(Globals):
-            fld = self._custom_field(tracker_globals)
-            tracker_globals.custom_fields.append(fld)
-        self.ormsession.flush()
-        self.ormsession.clear()
-
-    def down(self):
-        for tracker_globals in self.ormsession.find(Globals):
-            fld = self._custom_field(tracker_globals)
-            if tracker_globals.custom_fields[-1] == fld:
-                tracker_globals.custom_fields.pop()
-        self.ormsession.flush()
-        self.ormsession.clear()
-
-class FixMilestonesAndTickets(TrackerMigration):
-    version = 5
-
-    def up(self):
-        for tracker_globals in self.ormsession.find(Globals):
-            for fld in tracker_globals.custom_fields:
-                if 'name' not in fld:
-                    fld.name = '_' + fld.label.lower()
-                if 'show_in_search' not in fld:
-                    fld.show_in_search = True
-        for ticket in self.ormsession.find(Ticket):
-            ticket.custom_fields['_milestone'] = ticket.milestone
-        self.ormsession.flush()
-        self.ormsession.clear()
-
-    
+if __name__ == '__main__':
+    main()

--- a/ForgeWiki/forgewiki/model/migrations.py
+++ b/scripts/migrations/013-update-ordinals.py
@@ -1,72 +1,67 @@
-import cPickle as pickle
-from itertools import chain
+import sys
+import logging
 
-import bson
-from ming.orm import state
 from pylons import c
+from ming.orm import session
+from ming.orm.ormsession import ThreadLocalORMSession
 
-from flyway import Migration
-from allura.model import Thread, AppConfig, ArtifactReference
-from forgewiki.model import Page
+from allura import model as M
 
-class WikiMigration(Migration):
+log = logging.getLogger('update-ordinals')
+log.addHandler(logging.StreamHandler(sys.stdout))
 
-    def __init__(self, *args, **kwargs):
-        super(WikiMigration, self).__init__(*args, **kwargs)
-        try:
-            c.project
-        except TypeError:
-            class EmptyClass(): pass
-            c._push_object(EmptyClass())
-            c.project = EmptyClass()
-            c.project._id = None
-            c.app = EmptyClass()
-            c.app.config = EmptyClass()
-            c.app.config.options = EmptyClass()
-            c.app.config.options.mount_point = None
+def main():
+    test = sys.argv[-1] == 'test'
+    num_projects_examined = 0
+    log.info('Examining all projects for mount order.')
+    for some_projects in chunked_project_iterator({}):
+        for project in some_projects:
+            c.project = project
+            mounts = project.ordered_mounts(include_search=True)
+
+            # ordered_mounts() means duplicate ordinals (if any) will be next to each other
+            duplicates_found = False
+            prev_ordinal = None
+            for mount in mounts:
+                if mount['ordinal'] == prev_ordinal:
+                    duplicates_found = True
+                    break
+                prev_ordinal = mount['ordinal']
+
+            if duplicates_found:
+                if test:
+                    log.info('Would renumber mounts for project "%s".' % project.shortname)
+                else:
+                    log.info('Renumbering mounts for project "%s".' % project.shortname)
+                    for i, mount in enumerate(mounts):
+                        if 'ac' in mount:
+                            mount['ac'].options['ordinal'] = i
+                        elif 'sub' in mount:
+                            mount['sub'].ordinal = i
+                    ThreadLocalORMSession.flush_all()
+
+            num_projects_examined += 1
+            session(project).clear()
+
+        log.info('%s projects examined.' % num_projects_examined)
+        ThreadLocalORMSession.flush_all()
+        ThreadLocalORMSession.close_all()
 
 
-class V0(WikiMigration):
-    '''Migrate Thread.artifact_id to Thread.artifact_reference'''
-    version = 0
+PAGESIZE=1024
 
-    def up(self):
-        for pg in self.ormsession.find(Page):
-            q1 = self.ormsession.find(Thread, dict(artifact_id=pg._id))
-            q2 = self.ormsession.find(Thread, {'artifact_reference.artifact_id':pg._id})
-            for t in chain(q1, q2):
-                t.artifact_reference = self._dump_ref(pg)
-                t.artifact_id = None
-                self.ormsession.update_now(t, state(t))
-        self.ormsession.flush()
+def chunked_project_iterator(q_project):
+    '''shamelessly copied from refresh-all-repos.py'''
+    page = 0
+    while True:
+        results = (M.Project.query
+                   .find(q_project)
+                   .skip(PAGESIZE*page)
+                   .limit(PAGESIZE)
+                   .all())
+        if not results: break
+        yield results
+        page += 1
 
-    def down(self):
-        for pg in self.ormsession.find(Page):
-            for t in self.ormsession.find(Thread, dict(artifact_reference=self._dump_ref(pg))):
-                t.artifact_id = pg._id
-                t.artifact_reference = None
-                self.ormsession.update_now(t, state(t))
-        self.ormsession.flush()
-
-
-    def _dump_ref(self, art):
-        app_config = self.ormsession.get(AppConfig, art.app_config_id)
-        return ArtifactReference(dict(
-            project_id=app_config.project_id,
-            mount_point=app_config.options.mount_point,
-            artifact_type=bson.Binary(pickle.dumps(art.__class__)),
-            artifact_id=art._id))
-
-
-class AddDeletedAttribute(WikiMigration):
-    version = 1
-
-    def up(self):
-        for pg in self.ormsession.find(Page):
-            pg.deleted = False
-        self.ormsession.flush()
-
-    def down(self):
-        for pg in self.ormsession.find(Page):
-            del pg.deleted
-        self.ormsession.flush()
+if __name__ == '__main__':
+    main()