|
a/scripts/tracker-rip.py |
|
b/scripts/tracker-rip.py |
| 1 |
#!/usr/bin/python
|
1 |
#!/usr/bin/python
|
|
|
2 |
import sys
|
|
|
3 |
import getpass
|
|
|
4 |
from urlparse import urljoin
|
| 2 |
|
5 |
|
| 3 |
from sys import stdout
|
6 |
from pyforge.lib import rest_api
|
| 4 |
import hmac, hashlib
|
|
|
| 5 |
from datetime import datetime
|
|
|
| 6 |
import urllib
|
|
|
| 7 |
from urlparse import urlparse, urljoin
|
|
|
| 8 |
import urllib2
|
|
|
| 9 |
import json
|
|
|
| 10 |
from formencode import variabledecode
|
|
|
| 11 |
|
7 |
|
| 12 |
def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'):
|
8 |
SRC_CRED=dict(
|
| 13 |
"""
|
9 |
api_key='b15b105ee580a8652616',
|
| 14 |
Returns a bytestring version of 's', encoded as specified in 'encoding'.
|
10 |
secret_key='d2a3315517ba81491ed7b7498636495023b8fb2d72e461457a20aa4b9e7ad032e9a8d07187f36661',
|
|
|
11 |
http_username=raw_input('LDAP username: '),
|
|
|
12 |
http_password=getpass.getpass('LDAP password: '))
|
|
|
13 |
SRC_SERVER='https://newforge.sf.geek.net/'
|
|
|
14 |
SRC_TOOL='/rest/p/forge/tickets/'
|
| 15 |
|
15 |
|
| 16 |
If strings_only is True, don't convert (some) non-string-like objects.
|
16 |
# Credentials for sf-overlords
|
|
|
17 |
DST_CRED=dict(
|
|
|
18 |
api_key='a4a88c67179137053d70',
|
|
|
19 |
secret_key='fcc48a0c31459e99a88cc42cdd7f908fad78b283ca30a86caac1ab65036ff71fc195a18e56534dc5')
|
|
|
20 |
DST_SERVER='http://sourceforge.net/'
|
|
|
21 |
DST_TOOL='/rest/p/forge/tickets/'
|
| 17 |
|
22 |
|
| 18 |
This function was borrowed from Django
|
23 |
FAKE_TICKET={
|
| 19 |
"""
|
24 |
u'created_date': u'2010-03-08 17:29:42.802000',
|
| 20 |
if strings_only and isinstance(s, (types.NoneType, int)):
|
25 |
u'assigned_to_id': u'',
|
| 21 |
return s
|
26 |
u'assigned_to': u'',
|
| 22 |
elif not isinstance(s, basestring):
|
27 |
u'custom_fields': {'_component':'', '_size':0, '_priority':'', '_type':''},
|
| 23 |
try:
|
28 |
u'description': u'Ticket was not present in source',
|
| 24 |
return str(s)
|
29 |
u'milestone': u'',
|
| 25 |
except UnicodeEncodeError:
|
30 |
u'reported_by': u'',
|
| 26 |
if isinstance(s, Exception):
|
31 |
u'reported_by_id': u'',
|
| 27 |
# An Exception subclass containing non-ASCII data that doesn't
|
32 |
u'status': u'closed',
|
| 28 |
# know how to print itself properly. We shouldn't raise a
|
33 |
u'sub_ids': [],
|
| 29 |
# further exception.
|
34 |
u'summary': u'Placeholder ticket',
|
| 30 |
return ' '.join([smart_str(arg, encoding, strings_only,
|
35 |
u'super_id': u'None'}
|
| 31 |
errors) for arg in s])
|
|
|
| 32 |
return unicode(s).encode(encoding, errors)
|
|
|
| 33 |
elif isinstance(s, unicode):
|
|
|
| 34 |
r = s.encode(encoding, errors)
|
|
|
| 35 |
return r
|
|
|
| 36 |
elif s and encoding != 'utf-8':
|
|
|
| 37 |
return s.decode('utf-8', errors).encode(encoding, errors)
|
|
|
| 38 |
else:
|
|
|
| 39 |
return s
|
|
|
| 40 |
|
36 |
|
| 41 |
def generate_smart_str(params):
|
37 |
def main():
|
| 42 |
for (key, value) in params:
|
38 |
src_cli = rest_api.RestClient(
|
| 43 |
yield smart_str(key), smart_str(value)
|
39 |
base_uri=SRC_SERVER,
|
|
|
40 |
**SRC_CRED)
|
|
|
41 |
dst_cli = rest_api.RestClient(
|
|
|
42 |
base_uri=DST_SERVER,
|
|
|
43 |
**DST_CRED)
|
|
|
44 |
src = TicketAPI(src_cli, SRC_TOOL)
|
|
|
45 |
dst = TicketAPI(dst_cli, DST_TOOL)
|
|
|
46 |
for ticket in src.iter_tickets(check=True):
|
|
|
47 |
print 'Migrating ticket %s:\n%s' % (ticket['ticket_num'], ticket)
|
|
|
48 |
print 'Create ticket on %s' % DST_SERVER
|
|
|
49 |
dst.create_ticket(ticket)
|
|
|
50 |
print 'Create discussion on %s' % DST_SERVER
|
|
|
51 |
src_thread = src.load_thread(ticket)
|
|
|
52 |
if not src_thread or not src_thread['posts']:
|
|
|
53 |
print '... no posts'
|
|
|
54 |
continue
|
|
|
55 |
dst_thread = dst.load_thread(ticket)
|
|
|
56 |
slug_map = {}
|
|
|
57 |
for post in src.iter_posts(src_thread):
|
|
|
58 |
print '... migrate post %s:\n%r' % (post['slug'], post['text'])
|
|
|
59 |
dst.create_post(dst_thread, post, slug_map)
|
| 44 |
|
60 |
|
| 45 |
def urlencode(params):
|
61 |
class TicketAPI(object):
|
| 46 |
"""
|
|
|
| 47 |
A version of Python's urllib.urlencode() function that can operate on
|
|
|
| 48 |
unicode strings. The parameters are first case to UTF-8 encoded strings and
|
|
|
| 49 |
then encoded as per normal.
|
|
|
| 50 |
"""
|
|
|
| 51 |
return urllib.urlencode([i for i in generate_smart_str(params)])
|
|
|
| 52 |
|
62 |
|
|
|
63 |
def __init__(self, client, path):
|
|
|
64 |
self.client = client
|
|
|
65 |
self.path = path
|
| 53 |
|
66 |
|
| 54 |
class Signer(object):
|
67 |
def iter_tickets(self, min_ticket=1, max_ticket=None, check=False):
|
|
|
68 |
if check:
|
|
|
69 |
tickets = self.client.request('GET', self.path)['tickets']
|
|
|
70 |
valid_tickets = set(t['ticket_num'] for t in tickets)
|
|
|
71 |
max_valid_ticket = max(valid_tickets)
|
|
|
72 |
cur_ticket = min_ticket
|
|
|
73 |
while True:
|
|
|
74 |
if check and cur_ticket not in valid_tickets:
|
|
|
75 |
if cur_ticket > max_valid_ticket: break
|
|
|
76 |
yield dict(FAKE_TICKET, ticket_num=cur_ticket)
|
|
|
77 |
cur_ticket += 1
|
|
|
78 |
continue
|
|
|
79 |
ticket = self.client.request('GET', self.ticket_path(cur_ticket))['ticket']
|
|
|
80 |
if ticket is None: break
|
|
|
81 |
yield ticket
|
|
|
82 |
cur_ticket += 1
|
|
|
83 |
if max_ticket and cur_ticket > max_ticket: break
|
| 55 |
|
84 |
|
| 56 |
def __init__(self, secret_key, api_key):
|
85 |
def load_thread(self, ticket):
|
| 57 |
self.secret_key = secret_key
|
86 |
discussion = self.client.request('GET', self.discussion_path())['discussion']
|
| 58 |
self.api_key = api_key
|
87 |
for thd in discussion['threads']:
|
|
|
88 |
if thd['subject'].startswith('#%d ' % ticket['ticket_num']):
|
|
|
89 |
break
|
|
|
90 |
else:
|
|
|
91 |
return None
|
|
|
92 |
thread = self.client.request(
|
|
|
93 |
'GET',self.thread_path(thd['_id']))['thread']
|
|
|
94 |
return thread
|
| 59 |
|
95 |
|
| 60 |
def __call__(self, path, params):
|
96 |
def iter_posts(self, thread):
|
| 61 |
params.append(('api_key', self.api_key))
|
97 |
for p in sorted(thread['posts'], key=lambda p:p['slug']):
|
| 62 |
params.append(('api_timestamp', datetime.utcnow().isoformat()))
|
98 |
post = self.client.request(
|
| 63 |
message = path + '?' + urlencode(sorted(params))
|
99 |
'GET', self.post_path(thread['_id'], p['slug']))['post']
|
| 64 |
digest = hmac.new(self.secret_key, message, hashlib.sha256).hexdigest()
|
100 |
yield post
|
| 65 |
params.append(('api_signature', digest))
|
|
|
| 66 |
return params
|
|
|
| 67 |
|
101 |
|
| 68 |
|
102 |
def create_ticket(self, ticket):
|
| 69 |
class TicketIterator(object):
|
103 |
ticket = dict(ticket, labels='')
|
| 70 |
|
104 |
ticket['description'] = 'Created by: %s\nCreated date: %s\nAssigned to:%s\n\n%s' % (
|
| 71 |
def __init__(self, secret_key, api_key, url, max_ticket, min_ticket=1):
|
105 |
ticket['reported_by'], ticket['created_date'], ticket['assigned_to'], ticket['description'])
|
| 72 |
self.sign = Signer(secret_key, api_key)
|
|
|
| 73 |
self.cur_ticket_num = min_ticket
|
|
|
| 74 |
self.max_ticket_num = max_ticket
|
|
|
| 75 |
self.url = url
|
|
|
| 76 |
|
|
|
| 77 |
def __iter__(self):
|
|
|
| 78 |
return self
|
|
|
| 79 |
|
|
|
| 80 |
def next(self):
|
|
|
| 81 |
if self.cur_ticket_num > self.max_ticket_num:
|
|
|
| 82 |
raise StopIteration
|
|
|
| 83 |
url = urljoin(self.url, str(self.cur_ticket_num))+'/'
|
|
|
| 84 |
self.cur_ticket_num += 1
|
|
|
| 85 |
params = self.sign(urlparse(url).path, [])
|
|
|
| 86 |
try:
|
|
|
| 87 |
f = urllib2.urlopen(url+'?'+urlencode(params))
|
|
|
| 88 |
except urllib2.HTTPError, e:
|
|
|
| 89 |
if e.code == 404:
|
|
|
| 90 |
raise StopIteration
|
|
|
| 91 |
else:
|
|
|
| 92 |
raise
|
|
|
| 93 |
ticket = json.loads(f.read())['ticket'] or {}
|
|
|
| 94 |
for bad_key in ('assigned_to_id', 'created_date', 'reported_by', 'reported_by_id', 'super_id', 'sub_ids', '_id'):
|
106 |
for bad_key in ('assigned_to_id', 'created_date', 'reported_by', 'reported_by_id', 'super_id', 'sub_ids', '_id'):
|
| 95 |
if bad_key in ticket:
|
107 |
if bad_key in ticket:
|
| 96 |
del ticket[bad_key]
|
108 |
del ticket[bad_key]
|
|
|
109 |
ticket.setdefault('labels', '')
|
|
|
110 |
ticket['custom_fields'].setdefault('_size', 0)
|
|
|
111 |
ticket['custom_fields'].setdefault('_priority', 'low')
|
|
|
112 |
ticket['custom_fields'].setdefault('_type', 'Bug')
|
|
|
113 |
ticket['custom_fields'].setdefault('_type', 'Component')
|
|
|
114 |
if ticket['custom_fields']['_size'] is None:
|
|
|
115 |
ticket['custom_fields']['_size'] = 0
|
|
|
116 |
if ticket['milestone'] not in ('backlog', 'public2', 'GA', 'post-GA'):
|
| 97 |
ticket['labels'] = ''
|
117 |
ticket['milestone'] = ''
|
| 98 |
return ticket
|
118 |
if ticket['status'] not in 'open in-progress code-review validation closed'.split():
|
|
|
119 |
ticket['status'] = 'open'
|
|
|
120 |
r = self.client.request('POST', self.new_ticket_path(), ticket_form=ticket)
|
|
|
121 |
self.client.request(
|
|
|
122 |
'POST', self.ticket_path(r['ticket']['ticket_num'], 'save'),
|
|
|
123 |
ticket_form=ticket)
|
| 99 |
|
124 |
|
|
|
125 |
def create_post(self, thread, post, slug_map):
|
|
|
126 |
text = 'Post by %s:\n%s' % (
|
|
|
127 |
post['author'], post['text'])
|
|
|
128 |
if '/' in post['slug']:
|
|
|
129 |
parent_post = slug_map[post['slug'].rsplit('/', 1)[0]]
|
|
|
130 |
new_post = self.client.request(
|
|
|
131 |
'POST', self.post_path(thread['_id'], parent_post, 'reply'),
|
|
|
132 |
text=text)['post']
|
|
|
133 |
else:
|
|
|
134 |
new_post = self.client.request(
|
|
|
135 |
'POST', self.thread_path(thread['_id'], 'new'),
|
|
|
136 |
text=text)['post']
|
|
|
137 |
slug_map[post['slug']] = new_post['slug']
|
|
|
138 |
return new_post
|
| 100 |
|
139 |
|
| 101 |
class TicketPoster(object):
|
140 |
def new_ticket_path(self):
|
|
|
141 |
return urljoin(self.path, 'new')
|
| 102 |
|
142 |
|
| 103 |
def __init__(self, secret_key, api_key, url):
|
143 |
def ticket_path(self, ticket_num, suffix=''):
|
| 104 |
self.sign = Signer(secret_key, api_key)
|
144 |
return urljoin(self.path, str(ticket_num)) + '/' + suffix
|
| 105 |
self.url = urljoin(url, 'new')
|
|
|
| 106 |
|
145 |
|
| 107 |
def __call__(self, ticket):
|
146 |
def discussion_path(self):
|
| 108 |
ticket = variabledecode.variable_encode(ticket, add_repetitions=False)
|
147 |
return '%s_discuss/' % (self.path)
|
| 109 |
params = [('ticket_form', json.dumps(ticket))]
|
|
|
| 110 |
params = self.sign(urlparse(self.url).path, params)
|
|
|
| 111 |
try:
|
|
|
| 112 |
f = urllib2.urlopen(self.url, urlencode(params))
|
|
|
| 113 |
except urllib2.HTTPError, e:
|
|
|
| 114 |
stdout.write(e.read())
|
|
|
| 115 |
|
148 |
|
|
|
149 |
def thread_path(self, thread_id, suffix=''):
|
|
|
150 |
return '%s_discuss/thread/%s/%s' % (self.path, thread_id, suffix)
|
| 116 |
|
151 |
|
| 117 |
def main():
|
152 |
def post_path(self, thread_id, post_slug, suffix=''):
|
| 118 |
pw_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
|
153 |
return '%s_discuss/thread/%s/%s/%s' % (self.path, thread_id, post_slug, suffix)
|
| 119 |
pw_mgr.add_password(None, 'https://newforge.sf.geek.net/', '<REALM USER>', '<REALM PASSWORD>')
|
|
|
| 120 |
auth_handler = urllib2.HTTPBasicAuthHandler(pw_mgr)
|
|
|
| 121 |
opener = urllib2.build_opener(auth_handler)
|
|
|
| 122 |
urllib2.install_opener(opener)
|
|
|
| 123 |
|
154 |
|
| 124 |
newforge = TicketIterator(
|
155 |
def pm(etype, value, tb): # pragma no cover
|
| 125 |
secret_key='<YOUR NEWFORGE SECRET KEY>',
|
156 |
import pdb, traceback
|
| 126 |
api_key='<YOUR NEWFORGE API KEY>',
|
157 |
try:
|
| 127 |
url='https://newforge.sf.geek.net/rest/p/forge/tickets/',
|
158 |
from IPython.ipapi import make_session; make_session()
|
| 128 |
# max_ticket=672,
|
159 |
from IPython.Debugger import Pdb
|
| 129 |
max_ticket=5,
|
160 |
sys.stderr.write('Entering post-mortem IPDB shell\n')
|
| 130 |
min_ticket=1)
|
161 |
p = Pdb(color_scheme='Linux')
|
|
|
162 |
p.reset()
|
|
|
163 |
p.setup(None, tb)
|
|
|
164 |
p.print_stack_trace()
|
|
|
165 |
sys.stderr.write('%s: %s\n' % ( etype, value))
|
|
|
166 |
p.cmdloop()
|
|
|
167 |
p.forget()
|
|
|
168 |
# p.interaction(None, tb)
|
|
|
169 |
except ImportError:
|
|
|
170 |
sys.stderr.write('Entering post-mortem PDB shell\n')
|
|
|
171 |
traceback.print_exception(etype, value, tb)
|
|
|
172 |
pdb.post_mortem(tb)
|
| 131 |
|
173 |
|
| 132 |
# testing with a demo project, update this URL to a new tracker in the forge project
|
174 |
sys.excepthook = pm
|
| 133 |
post_to_production = TicketPoster(
|
|
|
| 134 |
secret_key='<YOUR SOURCEFORGE SECRET KEY>',
|
|
|
| 135 |
api_key='<YOUR SOURCEFORGE API KEY>',
|
|
|
| 136 |
url='https://sourceforge.net/rest/p/wolftest/newtix/')
|
|
|
| 137 |
|
|
|
| 138 |
for ticket in newforge:
|
|
|
| 139 |
post_to_production(ticket)
|
|
|
| 140 |
|
|
|
| 141 |
|
175 |
|
| 142 |
if __name__ == '__main__':
|
176 |
if __name__ == '__main__':
|
| 143 |
main()
|
177 |
main()
|