|
a/scripts/teamforge-import.py |
|
b/scripts/teamforge-import.py |
|
... |
|
... |
| 7 |
import os.path
|
7 |
import os.path
|
| 8 |
from time import mktime
|
8 |
from time import mktime
|
| 9 |
import json
|
9 |
import json
|
| 10 |
from urlparse import urlparse
|
10 |
from urlparse import urlparse
|
| 11 |
from urllib import FancyURLopener
|
11 |
from urllib import FancyURLopener
|
| 12 |
from pprint import pprint
|
|
|
| 13 |
from datetime import datetime
|
12 |
from datetime import datetime
|
|
|
13 |
from ConfigParser import ConfigParser
|
| 14 |
|
14 |
|
| 15 |
from suds.client import Client
|
15 |
from suds.client import Client
|
| 16 |
from suds import WebFault
|
16 |
from suds import WebFault
|
| 17 |
from ming.orm.ormsession import ThreadLocalORMSession
|
17 |
from ming.orm.ormsession import ThreadLocalORMSession
|
| 18 |
from ming.base import Object
|
18 |
from ming.base import Object
|
|
... |
|
... |
| 32 |
'''
|
32 |
'''
|
| 33 |
|
33 |
|
| 34 |
options = None
|
34 |
options = None
|
| 35 |
s = None # security token
|
35 |
s = None # security token
|
| 36 |
users = set()
|
36 |
users = set()
|
|
|
37 |
CONFIG_FILENAME='teamforge-import.cfg'
|
| 37 |
|
38 |
|
| 38 |
def make_client(api_url, app):
|
39 |
def make_client(api_url, app):
|
| 39 |
return Client(api_url + app + '?wsdl', location=api_url + app)
|
40 |
return Client(api_url + app + '?wsdl', location=api_url + app)
|
| 40 |
|
41 |
|
| 41 |
def main():
|
42 |
def main():
|
| 42 |
global options, s
|
43 |
global options, s
|
| 43 |
optparser = OptionParser(usage='''%prog [--options] [projID projID projID]\nIf no project ids are given, all projects will be migrated''')
|
44 |
config = ConfigParser({
|
|
|
45 |
'api-url':None,
|
|
|
46 |
'attachment-url':'/sf/%s/do/%s/',
|
|
|
47 |
'default-wiki-text':'PRODUCT NAME HERE',
|
|
|
48 |
'username':None,
|
|
|
49 |
'password':None,
|
|
|
50 |
'output-dir':'teamforge-export/',
|
|
|
51 |
'list-project-ids':'false',
|
|
|
52 |
'neighborhood':None,
|
|
|
53 |
'neighborhood-shortname':None,
|
|
|
54 |
'skip-frs-download':'false',
|
|
|
55 |
'skip-unsupported-check':'false'
|
|
|
56 |
})
|
|
|
57 |
if os.path.exists('teamforge-import.cfg'):
|
|
|
58 |
config.read(CONFIG_FILENAME)
|
|
|
59 |
|
|
|
60 |
optparser = OptionParser(
|
|
|
61 |
usage=('%prog [--options] [projID projID projID]\n'
|
|
|
62 |
'If no project ids are given, all projects will be migrated'))
|
|
|
63 |
|
|
|
64 |
# Command-line-only options
|
|
|
65 |
optparser.add_option(
|
|
|
66 |
'--extract-only', action='store_true', dest='extract',
|
|
|
67 |
help='Store data from the TeamForge API on the local filesystem; not load into Allura')
|
|
|
68 |
optparser.add_option(
|
|
|
69 |
'--load-only', action='store_true', dest='load',
|
|
|
70 |
help='Load into Allura previously-extracted data')
|
|
|
71 |
|
|
|
72 |
# Command-line options with defaults in config file
|
|
|
73 |
optparser.add_option(
|
| 44 |
optparser.add_option('--api-url', dest='api_url', help='e.g. https://hostname/ce-soap50/services/')
|
74 |
'--api-url', dest='api_url', help='e.g. https://hostname/ce-soap50/services/',
|
| 45 |
optparser.add_option('--attachment-url', dest='attachment_url', default='/sf/%s/do/%s/')
|
75 |
default=config.get('Extract', 'api-url'))
|
| 46 |
optparser.add_option('--default-wiki-text', dest='default_wiki_text', default='PRODUCT NAME HERE', help='used in determining if a wiki page text is default or changed')
|
76 |
optparser.add_option(
|
| 47 |
optparser.add_option('-u', '--username', dest='username')
|
77 |
'--attachment-url', dest='attachment_url',
|
| 48 |
optparser.add_option('-p', '--password', dest='password')
|
78 |
default=config.get('Load', 'attachment-url'))
|
| 49 |
optparser.add_option('-o', '--output-dir', dest='output_dir', default='teamforge-export/')
|
79 |
optparser.add_option(
|
|
|
80 |
'--default-wiki-text', dest='default_wiki_text',
|
|
|
81 |
help='used in determining if a wiki page text is default or changed',
|
|
|
82 |
default=config.get('Extract', 'default-wiki-text'))
|
|
|
83 |
optparser.add_option(
|
|
|
84 |
'-u', '--username', dest='username',
|
|
|
85 |
default=config.get('Extract', 'username'))
|
|
|
86 |
optparser.add_option(
|
|
|
87 |
'-p', '--password', dest='password',
|
|
|
88 |
default=config.get('Extract', 'password'))
|
|
|
89 |
optparser.add_option(
|
|
|
90 |
'-o', '--output-dir', dest='output_dir',
|
|
|
91 |
default=config.get('Extract', 'output-dir'))
|
|
|
92 |
optparser.add_option(
|
| 50 |
optparser.add_option('--list-project-ids', action='store_true', dest='list_project_ids')
|
93 |
'--list-project-ids', action='store_true', dest='list_project_ids',
|
| 51 |
optparser.add_option('--extract-only', action='store_true', dest='extract', help='Store data from the TeamForge API on the local filesystem; not load into Allura')
|
94 |
default=config.getboolean('Extract', 'list-project-ids'))
|
| 52 |
optparser.add_option('--load-only', action='store_true', dest='load', help='Load into Allura previously-extracted data')
|
95 |
optparser.add_option(
|
| 53 |
optparser.add_option('-n', '--neighborhood', dest='neighborhood', help='Neighborhood full name, to load in to')
|
96 |
'-n', '--neighborhood', dest='neighborhood',
|
| 54 |
optparser.add_option('--n-shortname', dest='neighborhood_shortname', help='Neighborhood shortname, for PFS extract SQL')
|
97 |
help='Neighborhood full name, to load in to',
|
|
|
98 |
default=config.get('Load', 'neighborhood'))
|
|
|
99 |
optparser.add_option(
|
|
|
100 |
'--n-shortname', dest='neighborhood_shortname',
|
|
|
101 |
help='Neighborhood shortname, for PFS extract SQL',
|
|
|
102 |
default=config.get('Load', 'neighborhood-shortname'))
|
|
|
103 |
optparser.add_option(
|
| 55 |
optparser.add_option('--skip-frs-download', action='store_true', dest='skip_frs_download')
|
104 |
'--skip-frs-download', action='store_true', dest='skip_frs_download',
|
|
|
105 |
default=config.getboolean('Extract', 'skip-frs-download'))
|
|
|
106 |
optparser.add_option(
|
| 56 |
optparser.add_option('--skip-unsupported-check', action='store_true', dest='skip_unsupported_check')
|
107 |
'--skip-unsupported-check', action='store_true', dest='skip_unsupported_check',
|
|
|
108 |
default=config.getboolean('Extract', 'skip-unsupported-check'))
|
| 57 |
options, project_ids = optparser.parse_args()
|
109 |
options, project_ids = optparser.parse_args()
|
| 58 |
|
110 |
|
| 59 |
# neither specified, so do both
|
111 |
# neither specified, so do both
|
| 60 |
if not options.extract and not options.load:
|
112 |
if not options.extract and not options.load:
|
| 61 |
options.extract = True
|
113 |
options.extract = True
|
| 62 |
options.load = True
|
114 |
options.load = True
|
| 63 |
|
115 |
|
| 64 |
|
|
|
| 65 |
if options.extract:
|
116 |
if options.extract:
|
| 66 |
c = make_client(options.api_url, 'CollabNet')
|
117 |
client = make_client(options.api_url, 'CollabNet')
|
| 67 |
api_v = c.service.getApiVersion()
|
118 |
api_v = client.service.getApiVersion()
|
| 68 |
if not api_v.startswith('5.4.'):
|
119 |
if not api_v.startswith('5.4.'):
|
| 69 |
log.warning('Unexpected API Version %s. May not work correctly.' % api_v)
|
120 |
log.warning('Unexpected API Version %s. May not work correctly.' % api_v)
|
| 70 |
|
121 |
|
| 71 |
s = c.service.login(options.username, options.password or getpass('Password: '))
|
122 |
s = client.service.login(options.username, options.password or getpass('Password: '))
|
| 72 |
teamforge_v = c.service.getVersion(s)
|
123 |
teamforge_v = client.service.getVersion(s)
|
| 73 |
if not teamforge_v.startswith('5.4.'):
|
124 |
if not teamforge_v.startswith('5.4.'):
|
| 74 |
log.warning('Unexpected TeamForge Version %s. May not work correctly.' % teamforge_v)
|
125 |
log.warning('Unexpected TeamForge Version %s. May not work correctly.' % teamforge_v)
|
| 75 |
|
126 |
|
| 76 |
if options.load:
|
127 |
if options.load:
|
| 77 |
if not options.neighborhood:
|
128 |
if not options.neighborhood:
|
|
... |
|
... |
| 87 |
|
138 |
|
| 88 |
if not project_ids:
|
139 |
if not project_ids:
|
| 89 |
if not options.extract:
|
140 |
if not options.extract:
|
| 90 |
log.error('You must specify project ids')
|
141 |
log.error('You must specify project ids')
|
| 91 |
return
|
142 |
return
|
| 92 |
projects = c.service.getProjectList(s)
|
143 |
projects = client.service.getProjectList(s)
|
| 93 |
project_ids = [p.id for p in projects.dataRows]
|
144 |
project_ids = [p.id for p in projects.dataRows]
|
| 94 |
|
145 |
|
| 95 |
if options.list_project_ids:
|
146 |
if options.list_project_ids:
|
| 96 |
print ' '.join(project_ids)
|
147 |
print ' '.join(project_ids)
|
| 97 |
return
|
148 |
return
|
|
... |
|
... |
| 99 |
if not os.path.exists(options.output_dir):
|
150 |
if not os.path.exists(options.output_dir):
|
| 100 |
os.makedirs(options.output_dir)
|
151 |
os.makedirs(options.output_dir)
|
| 101 |
for pid in project_ids:
|
152 |
for pid in project_ids:
|
| 102 |
if options.extract:
|
153 |
if options.extract:
|
| 103 |
try:
|
154 |
try:
|
| 104 |
project = c.service.getProjectData(s, pid)
|
155 |
project = client.service.getProjectData(s, pid)
|
| 105 |
log.info('Project: %s %s %s' % (project.id, project.title, project.path))
|
156 |
log.info('Project: %s %s %s' % (project.id, project.title, project.path))
|
| 106 |
out_dir = os.path.join(options.output_dir, project.id)
|
157 |
out_dir = os.path.join(options.output_dir, project.id)
|
| 107 |
if not os.path.exists(out_dir):
|
158 |
if not os.path.exists(out_dir):
|
| 108 |
os.mkdir(out_dir)
|
159 |
os.mkdir(out_dir)
|
| 109 |
|
160 |
|
| 110 |
get_project(project, c)
|
161 |
get_project(project, client)
|
| 111 |
get_files(project)
|
162 |
get_files(project)
|
| 112 |
get_homepage_wiki(project)
|
163 |
get_homepage_wiki(project)
|
| 113 |
get_discussion(project)
|
164 |
get_discussion(project)
|
| 114 |
get_news(project)
|
165 |
get_news(project)
|
| 115 |
if not options.skip_unsupported_check:
|
166 |
if not options.skip_unsupported_check:
|
|
... |
|
... |
| 126 |
if options.extract:
|
177 |
if options.extract:
|
| 127 |
log.info('Users encountered: %s', len(users))
|
178 |
log.info('Users encountered: %s', len(users))
|
| 128 |
with open(os.path.join(options.output_dir, 'usernames.json'), 'w') as out:
|
179 |
with open(os.path.join(options.output_dir, 'usernames.json'), 'w') as out:
|
| 129 |
out.write(json.dumps(list(users)))
|
180 |
out.write(json.dumps(list(users)))
|
| 130 |
|
181 |
|
| 131 |
def get_project(project, c):
|
182 |
def get_project(project, client):
|
| 132 |
cats = make_client(options.api_url, 'CategorizationApp')
|
183 |
cats = make_client(options.api_url, 'CategorizationApp')
|
| 133 |
|
184 |
|
| 134 |
data = c.service.getProjectData(s, project.id)
|
185 |
data = client.service.getProjectData(s, project.id)
|
| 135 |
access_level = { 1: 'public', 4: 'private', 3: 'gated community'}[
|
186 |
access_level = { 1: 'public', 4: 'private', 3: 'gated community'}[
|
| 136 |
c.service.getProjectAccessLevel(s, project.id)
|
187 |
client.service.getProjectAccessLevel(s, project.id)
|
| 137 |
]
|
188 |
]
|
| 138 |
admins = c.service.listProjectAdmins(s, project.id).dataRows
|
189 |
admins = client.service.listProjectAdmins(s, project.id).dataRows
|
| 139 |
members = c.service.getProjectMemberList(s, project.id).dataRows
|
190 |
members = client.service.getProjectMemberList(s, project.id).dataRows
|
| 140 |
groups = c.service.getProjectGroupList(s, project.id).dataRows
|
191 |
groups = client.service.getProjectGroupList(s, project.id).dataRows
|
| 141 |
categories = cats.service.getProjectCategories(s, project.id).dataRows
|
192 |
categories = cats.service.getProjectCategories(s, project.id).dataRows
|
| 142 |
save(json.dumps(dict(
|
193 |
save(json.dumps(dict(
|
| 143 |
data = dict(data),
|
194 |
data = dict(data),
|
| 144 |
access_level = access_level,
|
195 |
access_level = access_level,
|
| 145 |
admins = map(dict, admins),
|
196 |
admins = map(dict, admins),
|