|
a/OSSEval/OpenSourceProject/doap.py |
|
b/OSSEval/OpenSourceProject/doap.py |
|
... |
|
... |
| 9 |
# OSSEval Copyright 2014 Bitergium SLL
|
9 |
# OSSEval Copyright 2014 Bitergium SLL
|
| 10 |
# SOS Open Source Copyright 2012 Roberto Galoppini
|
10 |
# SOS Open Source Copyright 2012 Roberto Galoppini
|
| 11 |
# Author: Davide Galletti
|
11 |
# Author: Davide Galletti
|
| 12 |
|
12 |
|
| 13 |
|
13 |
|
| 14 |
|
14 |
import urllib2, json
|
| 15 |
from utils import Configuration, Forges
|
15 |
from utils import Configuration, Forges, StringList, StringHelper
|
| 16 |
|
16 |
|
| 17 |
from django.db import connections
|
17 |
from django.db import connections
|
|
|
18 |
|
| 18 |
|
19 |
|
| 19 |
class Apache():
|
20 |
class Apache():
|
| 20 |
|
21 |
|
| 21 |
@staticmethod
|
22 |
@staticmethod
|
| 22 |
def search(name):
|
23 |
def search(name):
|
|
... |
|
... |
| 24 |
cursor = connections['flossmole'].cursor()
|
25 |
cursor = connections['flossmole'].cursor()
|
| 25 |
parameters = { 'limit': Configuration.max_number_of_records }
|
26 |
parameters = { 'limit': Configuration.max_number_of_records }
|
| 26 |
cursor.execute("SELECT idDoap, Name, Shortdesc, Description, Homepage FROM apache_doap WHERE LOWER(name) LIKE '%%" + name.lower() + "%%' LIMIT %(limit)s", parameters)
|
27 |
cursor.execute("SELECT idDoap, Name, Shortdesc, Description, Homepage FROM apache_doap WHERE LOWER(name) LIKE '%%" + name.lower() + "%%' LIMIT %(limit)s", parameters)
|
| 27 |
results = cursor.fetchall()
|
28 |
results = cursor.fetchall()
|
| 28 |
for record in results:
|
29 |
for record in results:
|
| 29 |
fp = FlossmoleProject()
|
30 |
dap = DoapApacheProject()
|
| 30 |
fp.id = record[0]
|
31 |
dap.id = record[0]
|
| 31 |
fp.id_forge = Forges.FSF
|
32 |
dap.id_forge = Forges.APACHE
|
| 32 |
fp.datasource_id = record[1]
|
33 |
dap.datasource_id = 0
|
| 33 |
fp.name = record[2]
|
34 |
dap.name = record[1]
|
|
|
35 |
dap.shortdesc = record[2]
|
| 34 |
fp.description = record[3]
|
36 |
dap.description = record[3]
|
|
|
37 |
dap.homepage = record[4]
|
| 35 |
project_list.append(fp)
|
38 |
project_list.append(dap)
|
| 36 |
return project_list
|
39 |
return project_list
|
| 37 |
|
40 |
|
|
|
41 |
|
| 38 |
@staticmethod
|
42 |
@staticmethod
|
| 39 |
def getProjectInfo(project_identifier):
|
43 |
def getProjectInfo(project_identifier):
|
| 40 |
info = {}
|
44 |
dep = DoapApacheProject()
|
|
|
45 |
dep.load_from_db(project_identifier)
|
|
|
46 |
return { "doap" : dep.getDict() }
|
|
|
47 |
|
|
|
48 |
class Eclipse():
|
|
|
49 |
|
|
|
50 |
@staticmethod
|
|
|
51 |
def search(name):
|
|
|
52 |
project_list = []
|
| 41 |
cursor = connections['flossmole'].cursor()
|
53 |
cursor = connections['flossmole'].cursor()
|
| 42 |
cursor.execute("SELECT max(datasource_id) FROM fsf_projects")
|
54 |
parameters = { 'limit': Configuration.max_number_of_records }
|
| 43 |
row = cursor.fetchone()
|
55 |
cursor.execute("SELECT idDoap, Name, Shortdesc, Description, Homepage FROM eclipse_doap WHERE LOWER(name) LIKE '%%" + name.lower() + "%%' LIMIT %(limit)s", parameters)
|
| 44 |
datasource_id = row[0]
|
|
|
| 45 |
parameters = { 'datasource_id': datasource_id, 'project_identifier': project_identifier }
|
|
|
| 46 |
cursor.execute("SELECT proj_long_name, desc_long, url, real_url, released_on, proj_num FROM fsf_projects WHERE proj_unixname=%(project_identifier)s and datasource_id=%(datasource_id)s", parameters)
|
|
|
| 47 |
result = cursor.fetchone()
|
|
|
| 48 |
info['name'] = result[0]
|
|
|
| 49 |
info['desc_long'] = result[1]
|
|
|
| 50 |
info['url'] = result[2]
|
|
|
| 51 |
info['real_url'] = result[3]
|
|
|
| 52 |
info['released_on'] = result[4]
|
|
|
| 53 |
proj_num = result[5]
|
|
|
| 54 |
|
|
|
| 55 |
info['licenses'] = []
|
|
|
| 56 |
parameters = { 'datasource_id': datasource_id, 'proj_num': proj_num }
|
|
|
| 57 |
cursor.execute("SELECT license FROM fsf_project_licenses WHERE proj_num=%(proj_num)s and datasource_id=%(datasource_id)s", parameters)
|
|
|
| 58 |
results = cursor.fetchall()
|
56 |
results = cursor.fetchall()
|
| 59 |
for record in results:
|
57 |
for record in results:
|
| 60 |
info['licenses'].append(record[0])
|
58 |
dep = DoapEclipseProject()
|
| 61 |
|
|
|
| 62 |
info['developers'] = []
|
|
|
| 63 |
cursor.execute("SELECT person_name, role, email FROM fsf_developer_projects WHERE proj_num=%(proj_num)s and datasource_id=%(datasource_id)s", parameters)
|
|
|
| 64 |
results = cursor.fetchall()
|
|
|
| 65 |
for record in results:
|
|
|
| 66 |
developer = {}
|
|
|
| 67 |
developer['person_name'] = record[0]
|
|
|
| 68 |
developer['role'] = record[1]
|
|
|
| 69 |
developer['email'] = record[2]
|
59 |
dep.id = record[0]
|
| 70 |
info['developers'].append(developer)
|
60 |
dep.id_forge = Forges.ECLIPSE
|
|
|
61 |
dep.datasource_id = 0
|
|
|
62 |
dep.name = record[1]
|
|
|
63 |
dep.shortdesc = record[2]
|
|
|
64 |
dep.description = record[3]
|
|
|
65 |
dep.homepage = record[4]
|
|
|
66 |
project_list.append(dep)
|
|
|
67 |
return project_list
|
| 71 |
|
68 |
|
| 72 |
info['categories'] = []
|
69 |
@staticmethod
|
| 73 |
parameters = { 'datasource_id': datasource_id, 'proj_num': proj_num }
|
70 |
def getProjectInfo(project_identifier):
|
| 74 |
cursor.execute("SELECT project_category_title FROM fsf_project_categories WHERE proj_num=%(proj_num)s and datasource_id=%(datasource_id)s", parameters)
|
71 |
dep = DoapEclipseProject()
|
| 75 |
results = cursor.fetchall()
|
72 |
dep.load_from_db(project_identifier)
|
| 76 |
for record in results:
|
|
|
| 77 |
info['categories'].append(record[0])
|
|
|
| 78 |
|
73 |
|
| 79 |
info['requirements'] = []
|
74 |
# I try to get metrics from Bitergia's Grimoire
|
| 80 |
cursor.execute("SELECT requirement, requirement_type FROM fsf_project_requirements WHERE proj_num=%(proj_num)s and datasource_id=%(datasource_id)s", parameters)
|
75 |
try:
|
| 81 |
results = cursor.fetchall()
|
76 |
# http://dashboard.castalia.camp/projects/modeling.emf.compare_metrics.json
|
| 82 |
for record in results:
|
77 |
response = urllib2.urlopen("http://dashboard.castalia.camp/projects/" + dep.name + "_metrics.json")
|
| 83 |
requirement = {}
|
78 |
metrics_json = response.read()
|
| 84 |
requirement['requirement'] = record[0]
|
79 |
metrics_decoded = json.loads(metrics_json)
|
| 85 |
requirement['requirement_type'] = record[1]
|
80 |
|
| 86 |
info['requirements'].append(requirement)
|
81 |
return { "doap" : dep.getDict(), "metrics_decoded": metrics_decoded }
|
| 87 |
|
82 |
except:
|
| 88 |
return info
|
83 |
return { "doap" : dep.getDict() }
|
| 89 |
|
|
|
| 90 |
|
84 |
|
| 91 |
class FoafPerson:
|
85 |
class FoafPerson:
|
| 92 |
def __init__(self, firstName, lastName, login = ""):
|
86 |
def __init__(self, firstName, lastName, login = ""):
|
| 93 |
# see issue 867 for details: http://markosproject.berlios.de/mantis/view.php?id=867
|
87 |
# see issue 867 for details: http://markosproject.berlios.de/mantis/view.php?id=867
|
| 94 |
# Apache data sometimes is wrongly encoded: 'utf8' codec can't decode byte 0xfc in position 9: invalid start byte
|
88 |
# Apache data sometimes is wrongly encoded: 'utf8' codec can't decode byte 0xfc in position 9: invalid start byte
|
|
... |
|
... |
| 137 |
if hasattr(self, "cvs_repository") and (not (self.cvs_repository is None)):
|
131 |
if hasattr(self, "cvs_repository") and (not (self.cvs_repository is None)):
|
| 138 |
self.cvs_repository.parent_id_doap = self._idDoap
|
132 |
self.cvs_repository.parent_id_doap = self._idDoap
|
| 139 |
if hasattr(self, "git_repository") and (not (self.git_repository is None)):
|
133 |
if hasattr(self, "git_repository") and (not (self.git_repository is None)):
|
| 140 |
self.git_repository.parent_id_doap = self._idDoap
|
134 |
self.git_repository.parent_id_doap = self._idDoap
|
| 141 |
|
135 |
|
| 142 |
|
|
|
| 143 |
def load_from_db(self, idDoap):
|
136 |
def load_from_db(self, idDoap):
|
| 144 |
self.idDoap = idDoap
|
137 |
self.idDoap = idDoap
|
| 145 |
try:
|
138 |
try:
|
| 146 |
# expression for modified, modified_release is due to a bug in mysqldb for BIT data type
|
139 |
cursor = connections['flossmole'].cursor()
|
| 147 |
cursor = CrawlerDatabase.execute_cursor("SELECT name, shortdesc, description, homepage, created, mailing_list, download_page, bug_database, platform, service_endpoint, audience, blog, IF(modified=1,1,0) as modified, old_homepage, category, license, download_mirror, wiki, programming_language, os, language, idDWBatch, idProject, IF(modified_release=1,1,0) as modified_release FROM Doap WHERE idDoap="+str(idDoap))
|
140 |
cursor.execute("SELECT name, shortdesc, description, homepage, created, mailing_list, download_page, bug_database, platform, service_endpoint, audience, blog, old_homepage, category, license, download_mirror, wiki, programming_language, os, language FROM " + self.table_prefix + "_doap WHERE idDoap="+str(idDoap))
|
| 148 |
result = cursor.fetchone()
|
141 |
result = cursor.fetchone()
|
| 149 |
#for each project in the batch
|
|
|
| 150 |
if (result is None):
|
142 |
if not (result is None):
|
| 151 |
#throw
|
|
|
| 152 |
pass
|
|
|
| 153 |
else:
|
|
|
| 154 |
self.name = result[0]
|
143 |
self.name = result[0]
|
| 155 |
self.shortdesc = result[1]
|
144 |
self.shortdesc = result[1]
|
| 156 |
self.description = result[2]
|
145 |
self.description = result[2]
|
| 157 |
self.homepage = result[3]
|
146 |
self.homepage = result[3]
|
| 158 |
self.created = str(result[4])
|
147 |
self.created = str(result[4])
|
|
... |
|
... |
| 161 |
self.bug_database = result[7]
|
150 |
self.bug_database = result[7]
|
| 162 |
self.platform = result[8]
|
151 |
self.platform = result[8]
|
| 163 |
self.service_endpoint = result[9]
|
152 |
self.service_endpoint = result[9]
|
| 164 |
self.audience = result[10]
|
153 |
self.audience = result[10]
|
| 165 |
self.blog = result[11]
|
154 |
self.blog = result[11]
|
| 166 |
self.modified = True if result[12] == b'\x00' else False
|
|
|
| 167 |
self.modified_release = True if result[23] == b'\x00' else False
|
|
|
| 168 |
self.old_homepage = StringList().load_base64(result[13]).plain
|
155 |
self.old_homepage = StringList().load_base64(result[12]).plain
|
| 169 |
self.category = StringList().load_base64(result[14]).plain
|
156 |
self.category = StringList().load_base64(result[13]).plain
|
| 170 |
self.license = StringList().load_base64(result[15]).plain
|
157 |
self.license = StringList().load_base64(result[14]).plain
|
| 171 |
self.download_mirror = StringList().load_base64(result[16]).plain
|
158 |
self.download_mirror = StringList().load_base64(result[15]).plain
|
| 172 |
self.wiki = StringList().load_base64(result[17]).plain
|
159 |
self.wiki = StringList().load_base64(result[16]).plain
|
| 173 |
self.programming_language = StringList().load_base64(result[18]).plain
|
160 |
self.programming_language = StringList().load_base64(result[17]).plain
|
| 174 |
self.os = StringList().load_base64(result[19]).plain
|
161 |
self.os = StringList().load_base64(result[18]).plain
|
| 175 |
self.language = StringList().load_base64(result[20]).plain
|
162 |
self.language = StringList().load_base64(result[19]).plain
|
| 176 |
self.idDWBatch = result[21]
|
|
|
| 177 |
self.idProject = result[22]
|
|
|
| 178 |
#DoapVersion Table
|
163 |
#DoapVersion Table
|
| 179 |
self.release = []
|
164 |
self.release = []
|
|
|
165 |
cur = connections['flossmole'].cursor()
|
| 180 |
cur = CrawlerDatabase.execute_cursor("SELECT platform, revision, file_release, created, name FROM DoapVersion WHERE idDoap=" + str(self.idDoap))
|
166 |
cur.execute("SELECT platform, revision, file_release, created, name FROM " + self.table_prefix + "_DoapVersion WHERE idDoap=" + str(self.idDoap))
|
| 181 |
results = cur.fetchall()
|
167 |
results = cur.fetchall()
|
| 182 |
for record in results:
|
168 |
for record in results:
|
| 183 |
dv = DoapVersion()
|
169 |
dv = DoapVersion()
|
| 184 |
dv.name = record[4]
|
170 |
dv.name = record[4]
|
| 185 |
dv.created = str(record[3])
|
171 |
dv.created = str(record[3])
|
| 186 |
dv.revision = record[1]
|
172 |
dv.revision = record[1]
|
| 187 |
dv.platform = record[0]
|
173 |
dv.platform = record[0]
|
| 188 |
dv.file_release = StringList().load_base64(record[2]).plain
|
174 |
dv.file_release = StringList().load_base64(record[2]).plain
|
| 189 |
self.release.append(dv)
|
175 |
self.release.append(dv)
|
| 190 |
#DoapRepository Table
|
176 |
#DoapRepository Table
|
|
|
177 |
cur = connections['flossmole'].cursor()
|
| 191 |
cur = CrawlerDatabase.execute_cursor("SELECT browse, anon_root, location, type FROM DoapRepository WHERE idDoap=" + str(self.idDoap))
|
178 |
cur.execute("SELECT browse, anon_root, location, type FROM " + self.table_prefix + "_DoapRepository WHERE idDoap=" + str(self.idDoap))
|
| 192 |
results = cur.fetchall()
|
179 |
results = cur.fetchall()
|
| 193 |
for record in results:
|
180 |
for record in results:
|
| 194 |
dr = DoapRepository(self)
|
181 |
dr = DoapRepository(self)
|
| 195 |
dr.browse = record[0]
|
182 |
dr.browse = record[0]
|
| 196 |
dr.anon_root = record[1]
|
183 |
dr.anon_root = record[1]
|
|
... |
|
... |
| 221 |
self.tester = []
|
208 |
self.tester = []
|
| 222 |
self.translator = []
|
209 |
self.translator = []
|
| 223 |
parameters = {
|
210 |
parameters = {
|
| 224 |
'idDoapProject': idDoap
|
211 |
'idDoapProject': idDoap
|
| 225 |
}
|
212 |
}
|
|
|
213 |
cur = connections['flossmole'].cursor()
|
| 226 |
cur = CrawlerDatabase.execute_cursor("SELECT fp.firstName, fp.lastName, fp.login, dpfp.idDoapRole FROM FoafPerson fp JOIN DoapProjectFoafPerson dpfp on fp.idFoafPerson=dpfp.idFoafPerson WHERE idDoapProject=%(idDoapProject)s", parameters)
|
214 |
cur.execute("SELECT fp.firstName, fp.lastName, fp.login, dpfp.idDoapRole FROM " + self.table_prefix + "_FoafPerson fp JOIN " + self.table_prefix + "_DoapProjectFoafPerson dpfp on fp.idFoafPerson=dpfp.idFoafPerson WHERE idDoapProject=%(idDoapProject)s", parameters)
|
| 227 |
results = cur.fetchall()
|
215 |
results = cur.fetchall()
|
| 228 |
for record in results:
|
216 |
for record in results:
|
| 229 |
fp = FoafPerson(record[0], record[1], record[2])
|
217 |
fp = FoafPerson(record[0], record[1], record[2])
|
| 230 |
idDoapRole = record[3]
|
218 |
idDoapRole = record[3]
|
| 231 |
if idDoapRole == 1:
|
219 |
if idDoapRole == 1:
|
|
... |
|
... |
| 239 |
elif idDoapRole == 5:
|
227 |
elif idDoapRole == 5:
|
| 240 |
self.translator.append(fp)
|
228 |
self.translator.append(fp)
|
| 241 |
elif idDoapRole == 6:
|
229 |
elif idDoapRole == 6:
|
| 242 |
self.helper.append(fp)
|
230 |
self.helper.append(fp)
|
| 243 |
except Exception, e:
|
231 |
except Exception, e:
|
| 244 |
Logger.error(str(e))
|
232 |
print(str(e))
|
|
|
233 |
|
|
|
234 |
def getDict(self):
|
|
|
235 |
dic_var = self.__dict__.copy()
|
|
|
236 |
dic_var['maintainer'] = []
|
|
|
237 |
for m in self.maintainer:
|
|
|
238 |
dic_var['maintainer'].append(m.__dict__)
|
|
|
239 |
dic_var['release'] = []
|
|
|
240 |
for r in self.release:
|
|
|
241 |
dic_var['release'].append(r.__dict__)
|
|
|
242 |
if hasattr(self, 'svn_repository'):
|
|
|
243 |
dic_var['svn_repository'] = self.svn_repository.__dict__
|
|
|
244 |
|
|
|
245 |
return dic_var
|
|
|
246 |
|
|
|
247 |
|
|
|
248 |
class DoapApacheProject(DoapProject):
|
|
|
249 |
def __init__(self):
|
|
|
250 |
super(DoapApacheProject, self).__init__()
|
|
|
251 |
self.table_prefix = "apache"
|
|
|
252 |
|
|
|
253 |
class DoapEclipseProject(DoapProject):
|
|
|
254 |
def __init__(self):
|
|
|
255 |
super(DoapEclipseProject, self).__init__()
|
|
|
256 |
self.table_prefix = "eclipse"
|