|
a/OSSEval/OpenSourceProject/utils.py |
|
b/OSSEval/OpenSourceProject/utils.py |
|
... |
|
... |
| 12 |
|
12 |
|
| 13 |
|
13 |
|
| 14 |
from xml.dom import minidom
|
14 |
from xml.dom import minidom
|
| 15 |
import urllib2
|
15 |
import urllib2
|
| 16 |
import OpenSourceProject
|
16 |
import OpenSourceProject
|
|
|
17 |
import base64
|
| 17 |
|
18 |
|
| 18 |
class Configuration():
|
19 |
class Configuration():
|
| 19 |
api_key = OpenSourceProject.ohloh_api_key
|
20 |
api_key = OpenSourceProject.ohloh_api_key
|
| 20 |
max_number_of_records = OpenSourceProject.record_limit
|
21 |
max_number_of_records = OpenSourceProject.record_limit
|
| 21 |
|
22 |
|
|
... |
|
... |
| 63 |
success = True
|
64 |
success = True
|
| 64 |
except Exception as ex:
|
65 |
except Exception as ex:
|
| 65 |
print('Error downloading ' + url_string + " - Attempt n.:" + str(n_attempts) + " - " + str(ex))
|
66 |
print('Error downloading ' + url_string + " - Attempt n.:" + str(n_attempts) + " - " + str(ex))
|
| 66 |
n_attempts = n_attempts + 1
|
67 |
n_attempts = n_attempts + 1
|
| 67 |
return ret
|
68 |
return ret
|
|
|
69 |
|
|
|
70 |
class StringList():
|
|
|
71 |
separator = " "
|
|
|
72 |
|
|
|
73 |
def __init__(self):
|
|
|
74 |
self.plain = []
|
|
|
75 |
self.base64_encoded = ""
|
|
|
76 |
|
|
|
77 |
def load_plain(self, strings):
|
|
|
78 |
self.plain = strings
|
|
|
79 |
separator = ""
|
|
|
80 |
for s in self.plain:
|
|
|
81 |
self.base64_encoded += separator + base64.b64encode(s)
|
|
|
82 |
separator = StringList.separator
|
|
|
83 |
return self
|
|
|
84 |
|
|
|
85 |
def load_base64(self, base64_encoded):
|
|
|
86 |
self.base64_encoded = base64_encoded
|
|
|
87 |
for s in self.base64_encoded.split(StringList.separator):
|
|
|
88 |
self.plain.append(base64.b64decode(s))
|
|
|
89 |
return self
|
|
|
90 |
|
|
|
91 |
def remove_empty_strings(self):
|
|
|
92 |
# self.load_plain(filter(lambda text: text.strip(), self.plain))
|
|
|
93 |
self.load_plain([text for text in self.plain if text.strip()])
|
|
|
94 |
|
|
|
95 |
class StringHelper():
|
|
|
96 |
@staticmethod
|
|
|
97 |
def removeNonAscii(thisString):
|
|
|
98 |
return "".join(filter(lambda x: ord(x)<128, thisString))
|
|
|
99 |
@staticmethod
|
|
|
100 |
def makeUnicodeSafe(thisString):
|
|
|
101 |
'''
|
|
|
102 |
It is probably equivalent to the above method
|
|
|
103 |
'''
|
|
|
104 |
while True:
|
|
|
105 |
try:
|
|
|
106 |
return unicode(thisString)
|
|
|
107 |
except UnicodeDecodeError as ex: #UnicodeDecodeError
|
|
|
108 |
thisString = thisString[0:ex.start] + thisString[ex.end:]
|