[sysadmin-bin] add initial DOAP class

[Olav Vitters <ovitters src gnome org>]

commit fbed115091abe618a8a81fdaf958e4fa3fc0c58e
Author: Olav Vitters <olav vitters nl>
Date:   Tue Jan 20 21:18:40 2015 +0100

    add initial DOAP class

 ftpadmin |  155 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 154 insertions(+), 1 deletions(-)
---
diff --git a/ftpadmin b/ftpadmin
index 0c145e4..47b3aa7 100755
--- a/ftpadmin
+++ b/ftpadmin
@@ -31,6 +31,9 @@ try:
 except ImportError:
     from StringIO import StringIO
 
+import urlparse
+import urllib2
+import grp
 
 script_path = os.path.realpath(os.path.abspath(sys.argv[0]))
 script_dir = os.path.dirname(script_path) + '/git'
@@ -282,6 +285,156 @@ class BasicInfo(object):
         ('ChangeLog', 'changes', 'ChangeLog')
     ]
     DIFF_FILES_DICT = dict([(a,(b,c)) for a,b,c in DIFF_FILES])
+
+class DOAP(BasicInfo):
+    JSONVERSION = 1
+
+    NS_DOAP = "http://usefulinc.com/ns/doap#";
+    #NS_FOAF = "http://xmlns.com/foaf/0.1/";
+    NS_GNOME = "http://api.gnome.org/doap-extensions#";
+
+    DOAP_URL = 'https://git.gnome.org/repositories.doap'
+    DOAP_CACHE = ''
+    GNOME_REPO = 'git://git.gnome.org/'
+
+    TARBALL_PATH_PREFIX = '/sources/'
+    TARBALL_HOSTNAME_SUFFIX = '.gnome.org'
+
+    PROPERTIES = ('description', 'shortdesc', 'name')
+    PROPERTIES_AS_LIST = ('bug-database', )
+
+
+    # http://www.artima.com/forums/flat.jsp?forum=122&thread=15024
+
+    def __init__(self):
+        self._did_doap = False
+
+    def doap(self):
+
+        # Get module maintainer data from DOAP file in Git
+        # Note: some data is still in MAINTAINERS files. These are converted
+        #       to DOAP information by scripts in gitadmin-bin module.
+
+        etag = None
+        last_modified = None
+
+        # XXX - unfinished
+        valid_uids = set(grp.getgrnam('olav').gr_mem)
+        # XXX
+        # XXX - unfinished
+        if not os.path.exists(self.jsonfile):
+            force_refresh = True
+
+        if not force_refresh:
+            j = json.load(open(self.jsonfile, 'rb'))
+            json_ver = j[0]
+            if json_ver == self.JSONVERSION:
+                json_ver, etag, last_modified, UID_TO_MODULES, TARBALL_TO_MODULE, info = j
+                if not len(info):
+                    force_refresh=True
+            elif json_ver > self.JSONVERSION:
+                print >>sys.stderr, "ERROR: Json newer than supported version, ignoring json"
+                force_refresh=True
+            else:
+                force_refresh=True
+
+        req = urllib2.Request(self.DOAP_URL)
+
+        if not force_refresh:
+            if etag:
+                    req.add_header("If-None-Match", etag)
+            if last_modified:
+                    req.add_header("If-Modified-Since", last_modified)
+
+        try:
+            # Always need to check if there's any newer information
+            url_handle = urllib2.urlopen(req)
+        except urllib2.HTTPError, e:
+            if e.code == '304':
+                pass
+            elif force_refresh:
+                print >>sys.stderr, "ERROR: Cannot read DOAP file and no old copy available"
+            else:
+                print >>sys.stderr, "WARNING: Cannot retrieve DOAP file; using old copy"
+        else:
+            etag, last_modified, UID_TO_MODULES, TARBALL_TO_MODULE, info = self._parse_url_handle(url_handle)
+
+        self._info = info
+        self._tarball_to_module = TARBALL_TO_MODULE
+        self._uid_to_module = UID_TO_MODULES
+
+        if changed:
+            # save the new information
+            self.write_json()
+
+    def _parse_url_handle(self, url_handle):
+        UID_TO_MODULES = {}
+        TARBALL_TO_MODULE = {}
+        MODULE_INFO = {}
+
+        TARBALL_PATH_PREFIX_LEN = len(self.TARBALL_PATH_PREFIX)
+
+        headers = url_handle.info()
+        etag = headers.getheader("ETag")
+        etag = headers.getheader("ETagsss")
+        last_modified = headers.getheader("Last-Modified")
+
+        nodes = semi_rdf.read_rdf(url_handle)
+        for node in nodes:
+            if node.name != (self.NS_DOAP, "Project"):
+                continue
+
+            repo = node.find_property((self.NS_DOAP, u'repository'))
+            modname = None
+            if isinstance(repo, semi_rdf.Node):
+                repo_loc = repo.find_property((self.NS_DOAP, u'location'))
+
+                if hasattr(repo_loc, 'startswith') and repo_loc.startswith(self.GNOME_REPO):
+                    modname = repo_loc[len(self.GNOME_REPO):]
+
+            # In case project is unknown or already defined, ignore it
+            if not modname or modname in MODULE_INFO:
+                continue
+
+            MODULE_INFO[modname] = {}
+
+            tarballs = [url.path[url.path.index(self.TARBALL_PATH_PREFIX) + 
TARBALL_PATH_PREFIX_LEN:].split('/')[0] for url in [urlparse.urlparse(url) for url in 
node.find_properties((self.NS_DOAP, u'download-page')) if isinstance(url, semi_rdf.UrlResource)] if 
self.TARBALL_PATH_PREFIX in url.path and url.hostname.endswith(self.TARBALL_HOSTNAME_SUFFIX)]
+
+            if tarballs:
+                MODULE_INFO[modname]['tarballs'] = set(tarballs)
+
+                for tarball in tarballs:
+                    TARBALL_TO_MODULE.setdefault(tarball, set()).add(modname)
+
+            maints = set()
+            for maint in node.find_properties((self.NS_DOAP, u'maintainer')):
+                if not isinstance(maint, semi_rdf.Node):
+                    continue
+
+                uid = maint.find_property((self.NS_GNOME, u'userid'))
+                if not isinstance(uid, basestring):
+                    continue
+
+                uid = str(uid)
+
+                maints.add(uid)
+
+                UID_TO_MODULES.setdefault(uid, set()).add(modname)
+
+            if maints:
+                MODULE_INFO[modname]['maintainers'] = maints
+
+            for prop in self.PROPERTIES:
+                val = node.find_property((self.NS_DOAP, prop))
+                if val is not None:  MODULE_INFO[modname][prop] = val
+
+            for prop in self.PROPERTIES_AS_LIST:
+                val = node.find_properties((self.NS_DOAP, prop))
+                if val: MODULE_INFO[modname][prop] = list(val)
+
+        return (etag, last_modified, UID_TO_MODULES, TARBALL_TO_MODULE, MODULE_INFO)
+
+
 class TarInfo(BasicInfo):
 
     def __init__(self, path, files=set()):
@@ -454,6 +607,7 @@ class TarInfo(BasicInfo):
 
         return diffs
 
+
 class DirectoryInfo(BasicInfo):
     JSONVERSION = 4
 
@@ -761,7 +915,6 @@ class ModuleInfo(DirectoryInfo):
                 yield property
 
     def get_bz_product_from_doap(self):
-        import urlparse
         for bz in self.get_from_doap('bug-database'):
             url = urlparse.urlparse(bz)
             if url.netloc == 'bugzilla.gnome.org':