opensourceprojects.eu / osp-allura / Diff of /scripts/teamforge-import.py

Diff of /scripts/teamforge-import.py [5ff1d6] .. [8a4ef4]

Switch to unified view


...
        admin_usernames.add(admin.userName)
        user = get_user(admin.userName)
        c.user = user
        pr = user.project_role(project)
        pr.roles = [ role_admin._id ]

    role_developer = M.ProjectRole.by_name('Developer', project)
    for member in data.members:
        if member.userName in admin_usernames:
            continue
        user = get_user(member.userName)
...
            save(text, project, 'wiki', path+'.markdown')
            for img_ref in find_image_references(text):
                filename = img_ref.split('/')[-1]
                download_file('wiki', project.path + '/wiki/' + img_ref, project.id, 'wiki', path, filename)

def _dir_sql(created_on, project, dir_name, rel_path):
    if not rel_path:
        parent_directory = "'1'"
    else:
        parent_directory = "(SELECT pfs_path FROM pfs_path WHERE path_name = '%s/')" % rel_path
    sql = """
    UPDATE pfs
      SET file_crtime = '%s'
      WHERE source_pk = (SELECT project.project FROM project WHERE project.project_name = '%s')
      AND source_table = 'project'
      AND pfs_type = 'd'
      AND pfs_name = '%s'
      AND parent_directory = %s;
    """ % (created_on, convert_project_shortname(project.path), dir_name, parent_directory)
    return sql

def get_files(project):
    frs = make_client(options.api_url, 'FrsApp')
    valid_pfs_filename = re.compile(r'(?![. ])[-_ +.,=#~@!()\[\]a-zA-Z0-9]+(?<! )$')
    pfs_output_dir = os.path.join(os.path.abspath(options.output_dir), 'PFS', convert_project_shortname(project.path))
    sql_updates = ''

    def handle_path(obj, prev_path):
        path_component = obj.title.strip().replace('/', ' ').replace('&','').replace(':','')
        path = os.path.join(prev_path, path_component)
        if not valid_pfs_filename.match(path_component):
...
                 project, 'frs', rel_path+'_details.json')

            for file in frs.service.getFrsFileList(s, rel.id).dataRows:
                details = frs.service.getFrsFileData(s, file.id)

                file_path = handle_path(file, rel_path)
                save(json.dumps(dict(file,
                                     lastModifiedBy=details.lastModifiedBy,
                                     lastModifiedDate=details.lastModifiedDate,
                                     ),
                                default=str),
                     project,
                     'frs',
                     file_path+'.json'
                     )

                download_file('frs', rel.path + '/' + file.id, pfs_output_dir, file_path)

                mtime = int(mktime(details.lastModifiedDate.timetuple()))
                os.utime(os.path.join(pfs_output_dir, file_path), (mtime, mtime))
            # PFS update sql for rel
            created_on = int(mktime(rel.createdOn.timetuple()))


            mtime = int(mktime(rel.lastModifiedOn.timetuple()))
            os.utime(os.path.join(pfs_output_dir, rel_path), (mtime, mtime))
            sql_updates += _dir_sql(created_on, project, rel.title.strip(), pkg_path)
        # PFS update sql for pkg
        created_on = int(mktime(pkg.createdOn.timetuple()))
        mtime = int(mktime(pkg.lastModifiedOn.timetuple()))
        os.utime(os.path.join(pfs_output_dir, pkg_path), (mtime, mtime))
        sql_updates += _dir_sql(created_on, project, pkg.title.strip(), '')
    # save pfs update sql for this project
    with open(os.path.join(options.output_dir, 'pfs_updates.sql'), 'a') as out:
        out.write('/* %s */' % project.id)
        out.write(sql_updates)

'''

for forum in discussion.service.getForumList(s, p.id).dataRows:
    print forum.title
    for topic in discussion.service.getTopicList(s, forum.id).dataRows:
        print '  ', topic.title
        for post in discussion.service.getPostList(s, topic.id).dataRows:

	a/scripts/teamforge-import.py		b/scripts/teamforge-import.py
	...		...
215	admin_usernames.add(admin.userName)	215	admin_usernames.add(admin.userName)
216	user = get_user(admin.userName)	216	user = get_user(admin.userName)
217	c.user = user	217	c.user = user
218	pr = user.project_role(project)	218	pr = user.project_role(project)
219	pr.roles = [ role_admin._id ]	219	pr.roles = [ role_admin._id ]
220
221	role_developer = M.ProjectRole.by_name('Developer', project)	220	role_developer = M.ProjectRole.by_name('Developer', project)
222	for member in data.members:	221	for member in data.members:
223	if member.userName in admin_usernames:	222	if member.userName in admin_usernames:
224	continue	223	continue
225	user = get_user(member.userName)	224	user = get_user(member.userName)
	...		...
566	save(text, project, 'wiki', path+'.markdown')	565	save(text, project, 'wiki', path+'.markdown')
567	for img_ref in find_image_references(text):	566	for img_ref in find_image_references(text):
568	filename = img_ref.split('/')[-1]	567	filename = img_ref.split('/')[-1]
569	download_file('wiki', project.path + '/wiki/' + img_ref, project.id, 'wiki', path, filename)	568	download_file('wiki', project.path + '/wiki/' + img_ref, project.id, 'wiki', path, filename)
570		569
		570	def _dir_sql(created_on, project, dir_name, rel_path):
		571	if not rel_path:
		572	parent_directory = "'1'"
		573	else:
		574	parent_directory = "(SELECT pfs_path FROM pfs_path WHERE path_name = '%s/')" % rel_path
		575	sql = """
		576	UPDATE pfs
		577	SET file_crtime = '%s'
		578	WHERE source_pk = (SELECT project.project FROM project WHERE project.project_name = '%s')
		579	AND source_table = 'project'
		580	AND pfs_type = 'd'
		581	AND pfs_name = '%s'
		582	AND parent_directory = %s;
		583	""" % (created_on, convert_project_shortname(project.path), dir_name, parent_directory)
		584	return sql
		585
571	def get_files(project):	586	def get_files(project):
572	frs = make_client(options.api_url, 'FrsApp')	587	frs = make_client(options.api_url, 'FrsApp')
573	valid_pfs_filename = re.compile(r'(?![. ])[-_ +.,=#~@!()\[\]a-zA-Z0-9]+(?<! )$')	588	valid_pfs_filename = re.compile(r'(?![. ])[-_ +.,=#~@!()\[\]a-zA-Z0-9]+(?<! )$')
574	pfs_output_dir = os.path.join(os.path.abspath(options.output_dir), 'PFS', convert_project_shortname(project.path))	589	pfs_output_dir = os.path.join(os.path.abspath(options.output_dir), 'PFS', convert_project_shortname(project.path))
		590	sql_updates = ''
575		591
576	def handle_path(obj, prev_path):	592	def handle_path(obj, prev_path):
577	path_component = obj.title.strip().replace('/', ' ').replace('&','').replace(':','')	593	path_component = obj.title.strip().replace('/', ' ').replace('&','').replace(':','')
578	path = os.path.join(prev_path, path_component)	594	path = os.path.join(prev_path, path_component)
579	if not valid_pfs_filename.match(path_component):	595	if not valid_pfs_filename.match(path_component):
	...		...
595	project, 'frs', rel_path+'_details.json')	611	project, 'frs', rel_path+'_details.json')
596		612
597	for file in frs.service.getFrsFileList(s, rel.id).dataRows:	613	for file in frs.service.getFrsFileList(s, rel.id).dataRows:
598	details = frs.service.getFrsFileData(s, file.id)	614	details = frs.service.getFrsFileData(s, file.id)
599		615
600	file_path = os.path.join(rel_path, file.title.strip())	616	file_path = handle_path(file, rel_path)
601	save(json.dumps(dict(file,	617	save(json.dumps(dict(file,
602	lastModifiedBy=details.lastModifiedBy,	618	lastModifiedBy=details.lastModifiedBy,
603	lastModifiedDate=details.lastModifiedDate,	619	lastModifiedDate=details.lastModifiedDate,
604	),	620	),
605	default=str),	621	default=str),
606	project,	622	project,
607	'frs',	623	'frs',
608	file_path+'.json'	624	file_path+'.json'
609	)	625	)
610	#'''
611	download_file('frs', rel.path + '/' + file.id, pfs_output_dir, file_path)	626	download_file('frs', rel.path + '/' + file.id, pfs_output_dir, file_path)
612	# TODO: createdOn
613	mtime = int(mktime(details.lastModifiedDate.timetuple()))	627	mtime = int(mktime(details.lastModifiedDate.timetuple()))
614	os.utime(os.path.join(pfs_output_dir, file_path), (mtime, mtime))	628	os.utime(os.path.join(pfs_output_dir, file_path), (mtime, mtime))
615		629	# PFS update sql for rel
616	# now set mtime on the way back up the tree (so it isn't clobbered):	630	created_on = int(mktime(rel.createdOn.timetuple()))
617
618	# TODO: createdOn
619	mtime = int(mktime(rel.lastModifiedOn.timetuple()))	631	mtime = int(mktime(rel.lastModifiedOn.timetuple()))
620	os.utime(os.path.join(pfs_output_dir, rel_path), (mtime, mtime))	632	os.utime(os.path.join(pfs_output_dir, rel_path), (mtime, mtime))
621	# TODO: createdOn	633	sql_updates += _dir_sql(created_on, project, rel.title.strip(), pkg_path)
		634	# PFS update sql for pkg
		635	created_on = int(mktime(pkg.createdOn.timetuple()))
622	mtime = int(mktime(pkg.lastModifiedOn.timetuple()))	636	mtime = int(mktime(pkg.lastModifiedOn.timetuple()))
623	os.utime(os.path.join(pfs_output_dir, pkg_path), (mtime, mtime))	637	os.utime(os.path.join(pfs_output_dir, pkg_path), (mtime, mtime))
624	#'''	638	sql_updates += _dir_sql(created_on, project, pkg.title.strip(), '')
		639	# save pfs update sql for this project
		640	with open(os.path.join(options.output_dir, 'pfs_updates.sql'), 'a') as out:
		641	out.write('/* %s */' % project.id)
		642	out.write(sql_updates)
625		643
626	'''	644	'''
627
628	for forum in discussion.service.getForumList(s, p.id).dataRows:	645	for forum in discussion.service.getForumList(s, p.id).dataRows:
629	print forum.title	646	print forum.title
630	for topic in discussion.service.getTopicList(s, forum.id).dataRows:	647	for topic in discussion.service.getTopicList(s, forum.id).dataRows:
631	print ' ', topic.title	648	print ' ', topic.title
632	for post in discussion.service.getPostList(s, topic.id).dataRows:	649	for post in discussion.service.getPostList(s, topic.id).dataRows: