Open Source Projects Europe

--- a/OSSEval/OSProject/fixtures/initial_data.yaml
+++ b/OSSEval/OpenSourceProject/fixtures/initial_data.yaml
@@ -1,55 +1,45 @@
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 1
   fields:
     name: "SF"
     url: "http://sourceforge.net"
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 2
   fields:
     name: "FM"
     url: "http://freecode.com/"
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 5
   fields:
     name: "FSF"
     url: "http://www.fsf.org/"
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 11
   fields:
     name: "GH"
     url: "http://github.com/"
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 12
   fields:
     name: "GC"
     url: "http://code.google.com"
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 100
   fields:
     name: "OHLOH"
     url: "http://www.ohloh.net/"
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 1000
   fields:
     name: "CodePlex"
     url: "http://codeplex.com"
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 1200
   fields:
     name: "Apache"
     url: "http://apache.org"
-- model: OSProject.Forge
+- model: OpenSourceProject.Forge
   pk: 1300
   fields:
     name: "Eclipse"
     url: "http://eclipse.org"
-- model: OSProject.OSSConfiguration
-  pk: 1
-  fields:
-    flossmole_locale: True
-    ohloh_api_key: qdD49UAt6Heu1pV9vkC0A
-    local_record_limit: 50
-    remote_record_limit: 30
-    ohloh_record_limit: 30
-    
-    

--- a/OSSEval/OSProject/migrations/0001_initial.py
+++ b/OSSEval/OpenSourceProject/ohloh.py
@@ -1,147 +1,145 @@
-# -*- coding: utf-8 -*-
-from south.utils import datetime_utils as datetime
-from south.db import db
-from south.v2 import SchemaMigration
-from django.db import models
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright 2014 Bitergium SLL
+
+from xml.dom import minidom
+from utils import Configuration, xmlElements, Forges, UrllibHelper
+from OSSEval.utils import TrivialJSONEncoder 
+
+class OhlohProxy():
+    
+    def search_json(self, name):
+        a = TrivialJSONEncoder().encode(OhlohProxy.search(name))
+        return a
+
+    @staticmethod
+    def search(name):
+        page = 1
+        print "Ohloh search - " + name + " - page " + str(page)
+        url_ohloh = "https://www.openhub.net/p.xml?api_key=" + Configuration.api_key + "&query=" + name + "&page="
+        xml = UrllibHelper.urlopen(url_ohloh + str(page))
+        xmldoc = minidom.parseString(xml)
+
+        items_available = xmlElements(xmldoc.getElementsByTagName('items_available')).firstValue()
+        status = xmlElements(xmldoc.getElementsByTagName('status')).firstValue()
+        project_list = []
+        while (status == 'success' and len(project_list) < min(Configuration.max_number_of_records, items_available)):
+            print "Ohloh search - " + name + " - page " + str(page) + " len(project_list): " + str(len(project_list))
+            page = page + 1
+            item_list = xmldoc.getElementsByTagName('project')
+            for s in item_list :
+                try:
+                    op = OhlohProject()
+                    op.id = xmlElements(s.getElementsByTagName('url_name')).firstValue()
+                    op.name = xmlElements(s.getElementsByTagName('name')).firstValue()
+                    op.description = xmlElements(s.getElementsByTagName('description')).firstValue()
+                    op.small_logo_url = xmlElements(s.getElementsByTagName('small_logo_url')).firstValue()
+                    op.homepage_url = xmlElements(s.getElementsByTagName('homepage_url')).firstValue()
+                    op.id_forge = Forges.OHLOH
+                    op.datasource_id = 0
+                    project_list.append(op)
+                except Exception as ex:
+                    print str(ex)
+            xml = UrllibHelper.urlopen(url_ohloh + str(page))
+            xmldoc = minidom.parseString(xml)
+            status = xmldoc.getElementsByTagName('status')[0].firstChild.nodeValue
+        
+        return project_list
+
+    @staticmethod
+    def getProjectInfo(project_identifier):
+        url_ohloh = "http://www.openhub.net/projects/" + project_identifier + ".xml?api_key=" + Configuration.api_key
+        xml = UrllibHelper.urlopen(url_ohloh)
+        xmldoc = minidom.parseString(xml)
+        info = {}
+        
+        status = xmlElements(xmldoc.getElementsByTagName('status')).firstValue()
+        if status == 'success':
+            info['id'] = project_identifier
+            info['project_identifier'] = project_identifier
+            info['widget_languages'] = '<script type="text/javascript" src="http://www.openhub.net/p/' + project_identifier + '/widgets/project_languages.js"></script>'
+            info['ohloh_url'] = "http://www.openhub.net/p/" + project_identifier
+            info['commits_spark'] = "http://www.openhub.net/p/" + project_identifier + "/analyses/latest/commits_spark.png";
+            info['created_at'] = xmlElements(xmldoc.getElementsByTagName('created_at')).firstValue()
+            info['updated_at'] = xmlElements(xmldoc.getElementsByTagName('updated_at')).firstValue()
+            info['description'] = xmlElements(xmldoc.getElementsByTagName('description')).firstValue()
+            info['homepage_url'] = xmlElements(xmldoc.getElementsByTagName('homepage_url')).firstValue()
+            info['download_url'] = xmlElements(xmldoc.getElementsByTagName('download_url')).firstValue()
+            info['medium_logo_url'] = xmlElements(xmldoc.getElementsByTagName('medium_logo_url')).firstValue()
+            info['small_logo_url'] = xmlElements(xmldoc.getElementsByTagName('small_logo_url')).firstValue()
+            info['user_count'] = xmlElements(xmldoc.getElementsByTagName('user_count')).firstValue()
+            info['average_rating'] = xmlElements(xmldoc.getElementsByTagName('average_rating')).firstValue()
+            info['rating_count'] = xmlElements(xmldoc.getElementsByTagName('rating_count')).firstValue()
+            info['review_count'] = xmlElements(xmldoc.getElementsByTagName('review_count')).firstValue()
+            info['twelve_month_contributor_count'] = xmlElements(xmldoc.getElementsByTagName('twelve_month_contributor_count')).firstValue()
+            info['total_code_lines'] = xmlElements(xmldoc.getElementsByTagName('total_code_lines')).firstValue()
+            info['main_language_name'] = xmlElements(xmldoc.getElementsByTagName('main_language_name')).firstValue()
+            info['licenses'] = []
+            license_list = xmldoc.getElementsByTagName('license')
+            for s in license_list :
+                info['licenses'].append(xmlElements(s.getElementsByTagName('nice_name')).firstValue())
+            info['languages'] = []
+            language_list = xmldoc.getElementsByTagName('language')
+            for l in language_list :
+                language = {}
+                language['project_identifier'] = l.firstChild.nodeValue
+                language['percentage'] = l.attributes['percentage'].value
+                language['color'] = l.attributes['color'].value
+                language['id'] = l.attributes['id'].value
+                info['languages'].append(language)
+                
+        url_ohloh = "http://www.openhub.net/projects/" + project_identifier + "/factoids.xml?api_key=" + Configuration.api_key
+        xml = UrllibHelper.urlopen(url_ohloh)
+        xmldoc = minidom.parseString(xml)
+        
+        status = xmlElements(xmldoc.getElementsByTagName('status')).firstValue()
+        info['factoids'] = []
+        if status == 'success':
+            factoid_list = xmldoc.getElementsByTagName('factoid')
+            for f in factoid_list :
+                factoid = {}
+                factoid['type'] = xmlElements(f.getElementsByTagName('type')).firstValue()
+                factoid['description'] = xmlElements(f.getElementsByTagName('description')).firstValue()
+                factoid['severity'] = xmlElements(f.getElementsByTagName('severity')).firstValue()
+                info['factoids'].append(factoid)
 
 
-class Migration(SchemaMigration):
+        url_ohloh = "http://www.openhub.net/projects/" + project_identifier + "/analyses/latest/size_facts.xml?api_key=" + Configuration.api_key
+        xml = UrllibHelper.urlopen(url_ohloh)
+        xmldoc = minidom.parseString(xml)
+        
+        info['size_fact'] = None
+        status = xmlElements(xmldoc.getElementsByTagName('status')).firstValue()
+        if status == 'success':
+            size_fact_list = xmldoc.getElementsByTagName('size_fact')
+            if len(size_fact_list) > 0:
+                sf = size_fact_list[len(size_fact_list) - 1]
+                size_fact = {}
+                size_fact['month'] = xmlElements(sf.getElementsByTagName('month')).firstValue()
+                size_fact['code'] = xmlElements(sf.getElementsByTagName('code')).firstValue()
+                size_fact['comments'] = xmlElements(sf.getElementsByTagName('comments')).firstValue()
+                size_fact['blanks'] = xmlElements(sf.getElementsByTagName('comment_ratio')).firstValue()
+                size_fact['commits'] = xmlElements(sf.getElementsByTagName('commits')).firstValue()
+                size_fact['comment_ratio'] = xmlElements(sf.getElementsByTagName('comment_ratio')).firstValue()
+                size_fact['man_months'] = xmlElements(sf.getElementsByTagName('man_months')).firstValue()
+                # I just need the last as it is the most recent
+                info['size_fact'] = size_fact 
 
-    def forwards(self, orm):
-        # Adding model 'OSProject'
-        db.create_table(u'OSProject_osproject', (
-            ('name', self.gf('django.db.models.fields.CharField')(max_length=200)),
-            ('instance', self.gf('django.db.models.fields.related.OneToOneField')(related_name='actual_instance', unique=True, primary_key=True, to=orm['analysis.Instance'])),
-        ))
-        db.send_create_signal(u'OSProject', ['OSProject'])
+        return info
 
-        # Adding model 'Forge'
-        db.create_table(u'OSProject_forge', (
-            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
-            ('name', self.gf('django.db.models.fields.CharField')(max_length=50)),
-            ('url', self.gf('django.db.models.fields.CharField')(max_length=200)),
-        ))
-        db.send_create_signal(u'OSProject', ['Forge'])
-
-        # Adding model 'OSProjectForge'
-        db.create_table(u'OSProject_osprojectforge', (
-            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
-            ('name', self.gf('django.db.models.fields.CharField')(max_length=200)),
-            ('forge', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['OSProject.Forge'])),
-            ('os_project', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['OSProject.OSProject'])),
-            ('identifier_in_forge', self.gf('django.db.models.fields.CharField')(max_length=200)),
-            ('datasource_id', self.gf('django.db.models.fields.IntegerField')()),
-        ))
-        db.send_create_signal(u'OSProject', ['OSProjectForge'])
-
-        # Adding model 'OSSConfiguration'
-        db.create_table(u'OSProject_ossconfiguration', (
-            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
-            ('flossmole_locale', self.gf('django.db.models.fields.BooleanField')()),
-            ('ohloh_api_key', self.gf('django.db.models.fields.CharField')(max_length=100)),
-            ('local_record_limit', self.gf('django.db.models.fields.IntegerField')()),
-            ('remote_record_limit', self.gf('django.db.models.fields.IntegerField')()),
-            ('ohloh_record_limit', self.gf('django.db.models.fields.IntegerField')()),
-        ))
-        db.send_create_signal(u'OSProject', ['OSSConfiguration'])
+        
+class OhlohProject():
+    id = 0
+    name = ""
+    description = ""
+    small_logo_url = ""
+    homepage_url = ""
+    id_forge = 0
+    datasource_id = 0
+    def load_information(self):
+        self.info = OhlohProxy.getProjectInfo(self.name)
 
 
-    def backwards(self, orm):
-        # Deleting model 'OSProject'
-        db.delete_table(u'OSProject_osproject')
 
-        # Deleting model 'Forge'
-        db.delete_table(u'OSProject_forge')
-
-        # Deleting model 'OSProjectForge'
-        db.delete_table(u'OSProject_osprojectforge')
-
-        # Deleting model 'OSSConfiguration'
-        db.delete_table(u'OSProject_ossconfiguration')
-
-
-    models = {
-        u'OSProject.forge': {
-            'Meta': {'object_name': 'Forge'},
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
-            'os_projects': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['OSProject.OSProject']", 'symmetrical': 'False', 'through': u"orm['OSProject.OSProjectForge']", 'blank': 'True'}),
-            'url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
-        },
-        u'OSProject.osproject': {
-            'Meta': {'object_name': 'OSProject'},
-            'instance': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'actual_instance'", 'unique': 'True', 'primary_key': 'True', 'to': u"orm['analysis.Instance']"}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'})
-        },
-        u'OSProject.osprojectforge': {
-            'Meta': {'object_name': 'OSProjectForge'},
-            'datasource_id': ('django.db.models.fields.IntegerField', [], {}),
-            'forge': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['OSProject.Forge']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'identifier_in_forge': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
-            'os_project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['OSProject.OSProject']"})
-        },
-        u'OSProject.ossconfiguration': {
-            'Meta': {'object_name': 'OSSConfiguration'},
-            'flossmole_locale': ('django.db.models.fields.BooleanField', [], {}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'local_record_limit': ('django.db.models.fields.IntegerField', [], {}),
-            'ohloh_api_key': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
-            'ohloh_record_limit': ('django.db.models.fields.IntegerField', [], {}),
-            'remote_record_limit': ('django.db.models.fields.IntegerField', [], {})
-        },
-        u'analysis.analysis': {
-            'Meta': {'object_name': 'Analysis'},
-            'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
-            'created': ('django.db.models.fields.DateField', [], {'default': 'datetime.datetime.now', 'blank': 'True'}),
-            'description': ('django.db.models.fields.CharField', [], {'max_length': '2000'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'methodology_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['methodology.MethodologyVersion']"}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
-            'protected': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
-            'user_login': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
-            'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
-            'weight_scenario': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['methodology.WeightScenario']"})
-        },
-        u'analysis.instance': {
-            'Meta': {'object_name': 'Instance'},
-            'analysis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['analysis.Analysis']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
-            'name_for_search': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '200'})
-        },
-        u'entity.entity': {
-            'Meta': {'object_name': 'Entity'},
-            'actual_entity_app': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
-            'actual_entity_class': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'})
-        },
-        u'methodology.methodology': {
-            'Meta': {'object_name': 'Methodology'},
-            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
-            'description': ('django.db.models.fields.CharField', [], {'max_length': '2000', 'null': 'True', 'blank': 'True'}),
-            'documentation': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
-            'entity': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['entity.Entity']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'})
-        },
-        u'methodology.methodologyversion': {
-            'Meta': {'object_name': 'MethodologyVersion'},
-            'created': ('django.db.models.fields.DateField', [], {}),
-            'current': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'methodology': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['methodology.Methodology']"}),
-            'number': ('django.db.models.fields.IntegerField', [], {})
-        },
-        u'methodology.weightscenario': {
-            'Meta': {'object_name': 'WeightScenario'},
-            'active': ('django.db.models.fields.BooleanField', [], {}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'methodology_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['methodology.MethodologyVersion']"}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'})
-        }
-    }
-
-    complete_apps = ['OSProject']

--- a/OSSEval/OSProject/models.py
+++ b/OSSEval/OpenSourceProject/models.py
@@ -9,7 +9,7 @@
 
 class OSProject(models.Model):
     """
-    Must be the same name of the Django app
+    Must be the same name of the Django app                         ??????????????????????????????????????
     """
     name = models.CharField(max_length=200)
     instance = models.OneToOneField('analysis.Instance', related_name='actual_instance', primary_key=True)
@@ -20,7 +20,7 @@
     def getInstanceInfo(self):
         '''
         Part of the required interface
-        i is a dictionary; each source information is withing the source name key, e.g.
+        i is a dictionary; each source information is within the source name key, e.g.
         i['FM']
         i['OHLOH']
         i['FSF']
@@ -55,6 +55,10 @@
         try: i['homepage'] = i['OHLOH']['homepage_url'] 
         except: pass 
 
+        # TODO: i['documentation_url'] i['faq_url'] i['wiki_url']
+        # i['numBugsTotal']
+        # svn was in Melquiades; is it in any other source? 
+        
         if i['homepage'] <> "":
             try:
                 i['siteforsearchengine'] = urlparse(i['homepage']).netloc
@@ -76,7 +80,26 @@
             i['FSF']['released_on'] = (date.today() - i['FSF']['released_on']).days
             if i['FSF']['days'] > i['days']: i['days'] = i['FSF']['days']
         except: pass 
+        try: 
+            i['download_url'] = i['OHLOH']['download_url']
+        except: pass 
 
+        # CLEANUP
+        try: i['FM']['popularity_score'] = (i['FM']['popularity_score']).replace('\n','').strip()
+        except: pass
+        try: i['FM']['rating'] = (i['FM']['rating']).replace('\n','').strip()
+        except: pass
+        try: i['FM']['rating_count'] = (i['FM']['rating_count']).replace('\n','').strip()
+        except: pass
+        try: i['FM']['url_bugtracker'] = (i['FM']['url_bugtracker']).replace('\n','').strip()
+        except: pass
+        try: 
+            if 'OHLOH' in i.keys() and 'size_fact' in i['OHLOH'].keys() and 'comment_ratio' in i['OHLOH']['size_fact'].keys():
+                fl = float(i['OHLOH']['size_fact']['comment_ratio'])
+                # _pp stands for pretty print
+                i['OHLOH']['size_fact']['comment_ratio_pp'] = str(round(fl*100,1))+"%"
+        except: pass
+                    
         return i
 
 
@@ -108,10 +131,3 @@
             osprojectforge_info = GH.getProjectInfo(self.identifier_in_forge)
         return osprojectforge_info
         
-        
-class OSSConfiguration(models.Model):
-    flossmole_locale = models.BooleanField()
-    ohloh_api_key = models.CharField(max_length = 100)
-    local_record_limit = models.IntegerField()
-    remote_record_limit = models.IntegerField()
-    ohloh_record_limit = models.IntegerField()