[sysadmin-bin] Allow generating DOAP_CACHE from GitLab API rather than from the former repositories.doap file (Clos

[Andrea Veri <averi src gnome org>]

commit c19b3540a160f31656f48f5417ab6bffa2b2a3ed
Author: Andrea Veri <averi redhat com>
Date:   Mon Aug 9 17:32:48 2021 +0200

    Allow generating DOAP_CACHE from GitLab API rather than from the former repositories.doap file (Closes: 
https://gitlab.gnome.org/Infrastructure/Infrastructure/-/issues/638)

 ftpadmin | 98 ++++++++++++++++++++++++++--------------------------------------
 1 file changed, 40 insertions(+), 58 deletions(-)
---
diff --git a/ftpadmin b/ftpadmin
index f05d3f3..69c63e3 100755
--- a/ftpadmin
+++ b/ftpadmin
@@ -343,57 +343,32 @@ class DOAP(BasicInfo):
             return sorted(modules, key=len)[0]
 
     def _init_doap(self, force_refresh=False):
+        from datetime import datetime
+        import dateutil.relativedelta as relativedelta
 
-        # Get module maintainer data from DOAP file in Git
-        # Note: some data is still in MAINTAINERS files. These are converted
-        #       to DOAP information by scripts in gitadmin-bin module.
-
-        changed = False
-        etag = None
-        last_modified = None
-
-        # XXX - unfinished
-        if not os.path.exists(self.jsonfile):
+        if not os.path.exists(self.jsonfile) or os.path.getsize(self.jsonfile) == 0:
             force_refresh = True
 
+        if os.path.exists(self.jsonfile) and datetime.fromtimestamp(os.path.getmtime(self.jsonfile)) < 
(datetime.now() - relativedelta.relativedelta(days=1)):            force_refresh = True
+
         if not force_refresh:
             j = json.load(open(self.jsonfile, 'rb'))
             json_ver = j[0]
             if json_ver == self.JSONVERSION:
-                json_ver, etag, last_modified, info, TARBALL_TO_MODULE, UID_TO_MODULES = j
+                json_ver, info, TARBALL_TO_MODULE, UID_TO_MODULES = j
                 if not len(info):
-                    force_refresh=True
+                    force_refresh = True
             elif json_ver > self.JSONVERSION:
                 print >>sys.stderr, "ERROR: Json newer than supported version, ignoring json"
-                force_refresh=True
+                force_refresh = True
             else:
-                force_refresh=True
-
-        req = urllib2.Request(self.DOAP_URL)
-
-        if not force_refresh:
-            if etag:
-                    req.add_header("If-None-Match", etag)
-            if last_modified:
-                    req.add_header("If-Modified-Since", last_modified)
+                force_refresh = True
 
-        try:
-            # Always need to check if there's any newer information
-            url_handle = urllib2.urlopen(req)
-        except urllib2.HTTPError, e:
-            if e.code == 304:
-                pass
-            elif force_refresh:
-                print >>sys.stderr, "ERROR: Cannot read DOAP file and no old copy available"
-            else:
-                print e.code
-                print >>sys.stderr, "WARNING: Cannot retrieve DOAP file; using old copy"
-        else:
-            etag, last_modified, info, TARBALL_TO_MODULE, UID_TO_MODULES = self._parse_url_handle(url_handle)
+        changed = False
+        if force_refresh:
+           info, TARBALL_TO_MODULE, UID_TO_MODULES = self._parse_url_handle()
             changed = True
 
-        self.etag = etag
-        self.last_modified = last_modified
         self.info = info
         self.tarball_to_module = TARBALL_TO_MODULE
         self.uid_to_module = UID_TO_MODULES
@@ -402,38 +377,45 @@ class DOAP(BasicInfo):
             # save the new information
             self.write_json()
 
-    def _parse_url_handle(self, url_handle):
+    def _parse_url_handle(self):
+        from xml.sax import SAXParseException
+        import gitlab
+
         UID_TO_MODULES = {}
         TARBALL_TO_MODULE = {}
         MODULE_INFO = {}
 
         TARBALL_PATH_PREFIX_LEN = len(self.TARBALL_PATH_PREFIX)
 
-        headers = url_handle.info()
-        etag = headers.getheader("ETag")
-        last_modified = headers.getheader("Last-Modified")
+        exec(open("/home/admin/secret/gitlab_rw").read())
 
-        nodes = semi_rdf.read_rdf(url_handle)
-        for node in nodes:
-            if node.name != (self.NS_DOAP, "Project"):
-                continue
+        gl = gitlab.Gitlab('https://gitlab.gnome.org', GITLAB_PRIVATE_RW_TOKEN, api_version=4)
+
+        group = gl.groups.get(8, with_projects=False)
+        projects = group.projects.list(all=True, with_shared=False)
 
-            repo = node.find_property((self.NS_DOAP, u'repository'))
-            modname = None
-            if isinstance(repo, semi_rdf.Node):
-                repo_loc = repo.find_property((self.NS_DOAP, u'location'))
+        doap_url = 'https://gitlab.gnome.org/%s/%s/raw/%s/%s.doap'
 
-                if hasattr(repo_loc, 'startswith'):
-                    modname = re.sub('%s:(GNOME|Infrastructure)/' % self.GITLAB_REPO, '', repo_loc)
-                    modname = modname.split('.')[0]
+        for p in projects:
+            default_branch = p.attributes.get('default_branch')
+            if not default_branch:
+               default_branch = 'master'
+            namespace = p.attributes['namespace']['name']
+            name = p.attributes.get('path')
 
-            # In case project is unknown or already defined, ignore it
-            if not modname or modname in MODULE_INFO:
+            try:
+                nodes = semi_rdf.read_rdf(doap_url % (namespace, name, default_branch, name))
+            except SAXParseException:
                 continue
 
-            MODULE_INFO[modname] = {}
+            for node in nodes:
+                if node.name != (self.NS_DOAP, "Project"):
+                    continue
+
+                modname = name
+                MODULE_INFO[modname] = {}
 
-            tarballs = [url.path[url.path.index(self.TARBALL_PATH_PREFIX) + 
TARBALL_PATH_PREFIX_LEN:].split('/')[0] for url in [urlparse.urlparse(url) for url in 
node.find_properties((self.NS_DOAP, u'download-page')) if isinstance(url, semi_rdf.UrlResource)] if 
self.TARBALL_PATH_PREFIX in url.path and url.hostname.endswith(self.TARBALL_HOSTNAME_SUFFIX)]
+                tarballs = [url.path[url.path.index(self.TARBALL_PATH_PREFIX) + 
TARBALL_PATH_PREFIX_LEN:].split('/')[0] for url in [urlparse.urlparse(url) for url in 
node.find_properties((self.NS_DOAP, u'download-page')) if isinstance(url, semi_rdf.UrlResource)] if 
self.TARBALL_PATH_PREFIX in url.path and url.hostname.endswith(self.TARBALL_HOSTNAME_SUFFIX)]
 
             if tarballs:
                 MODULE_INFO[modname]['tarballs'] = set(tarballs)
@@ -467,14 +449,14 @@ class DOAP(BasicInfo):
                 val = node.find_properties((self.NS_DOAP, prop))
                 if val: MODULE_INFO[modname][prop] = list(val)
 
-        return (etag, last_modified, MODULE_INFO, TARBALL_TO_MODULE, UID_TO_MODULES)
+        return (MODULE_INFO, TARBALL_TO_MODULE, UID_TO_MODULES)
 
     def write_json(self):
         # Want to overwrite any existing file and change the owner
         if os.path.exists(self.jsonfile):
             os.remove(self.jsonfile)
         with open(self.jsonfile, 'w') as f:
-            json.dump((self.JSONVERSION, self.etag, self.last_modified, self.info, self.tarball_to_module, 
self.uid_to_module), f, cls=SetEncoder)
+            json.dump((self.JSONVERSION, self.info, self.tarball_to_module, self.uid_to_module), f, 
cls=SetEncoder)
             if self.GROUPID is not None:
                 os.fchown(f.fileno(), -1, self.GROUPID)