[odrs-web] Dedupe the list of components whenever we check the database

[Richard Hughes <rhughes src gnome org>]

commit 173b0b294baf6a6777b4901f15b84266d9d08e2f
Author: Richard Hughes <richard hughsie com>
Date:   Mon Jul 8 08:46:55 2019 +0100

    Dedupe the list of components whenever we check the database

 Dockerfile                                    |  2 +
 app_data/cron.py                              | 63 +++++++++++++++++++++++++--
 app_data/httpd-pre-init/entrypoint.sh         |  2 +-
 app_data/migrations/versions/ef03b3a98056_.py | 27 ------------
 4 files changed, 63 insertions(+), 31 deletions(-)
---
diff --git a/Dockerfile b/Dockerfile
index 144144e..0eddaee 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -37,6 +37,8 @@ RUN LANG=en_US.utf8 \
     SQLALCHEMY_TRACK_MODIFICATIONS=False \
     flask db upgrade
 
+RUN python36 /opt/app-root/src/cron.py fsck
+
 RUN curl https://flathub.org/repo/appstream/x86_64/appstream.xml.gz -o /tmp/appstream.xml.gz
 
 RUN python36 /opt/app-root/src/cron.py appstream-import /tmp/appstream.xml.gz
diff --git a/app_data/cron.py b/app_data/cron.py
index 465cc7a..cc035cf 100755
--- a/app_data/cron.py
+++ b/app_data/cron.py
@@ -20,6 +20,59 @@ from odrs import db
 from odrs.models import Review, Taboo, Component
 from odrs.util import _get_rating_for_component, _get_taboos_for_locale
 
+def _fsck_components():
+
+    # get all existing components
+    components = {}
+    for component in db.session.query(Component).\
+                        filter(Component.app_id != '').\
+                        order_by(Component.app_id.asc()).all():
+        components[component.app_id] = component
+
+    # guessed, thanks Canonical :/
+    for app_id in components:
+        if not app_id.startswith('io.snapcraft.'):
+            continue
+        if components[app_id].component_id_parent:
+            continue
+        name, _ = app_id[13:].rsplit('-', maxsplit=1)
+        parent = components.get(name + '.desktop')
+        if not parent:
+            continue
+        print('adding snapcraft parent for {} -> {}'.format(components[app_id].app_id,
+                                                            parent.app_id))
+        parent.adopt(components[app_id])
+
+    # upstream drops the .desktop sometimes
+    for app_id in components:
+        if components[app_id].component_id_parent:
+            continue
+        app_id_new = app_id.replace('.desktop', '')
+        if app_id == app_id_new:
+            continue
+        parent = components.get(app_id_new)
+        if not parent:
+            continue
+        print('adding parent for {} -> {}'.format(components[app_id].app_id,
+                                                  parent.app_id))
+        parent.adopt(components[app_id])
+
+    # API change :/
+    for app_id in components:
+        if not app_id.endswith('.shell-extension'):
+            continue
+        if components[app_id].component_id_parent:
+            continue
+        app_id_new = app_id.replace('.shell-extension', '')
+        app_id_new = app_id_new.replace('@', '_')
+        parent = components.get(app_id_new)
+        if not parent:
+            continue
+        print('adding shell parent for {} -> {}'.format(components[app_id].app_id,
+                                                        parent.app_id))
+        parent.adopt(components[app_id])
+    db.session.commit()
+
 def _auto_delete(days=31):
 
     since = datetime.datetime.now() - datetime.timedelta(days=days)
@@ -36,6 +89,10 @@ def _auto_delete(days=31):
         db.session.delete(review)
     db.session.commit()
 
+def _fsck():
+    _auto_delete()
+    _fsck_components()
+
 def _regenerate_ratings(fn):
     item = {}
     for component in db.session.query(Component).\
@@ -153,7 +210,7 @@ def _taboo_import(fn):
 if __name__ == '__main__':
 
     if len(sys.argv) < 2:
-        print('Usage: %s ratings|auto-delete|taboo-check|taboo-import' % sys.argv[0])
+        print('Usage: %s ratings|fsck|taboo-check|taboo-import' % sys.argv[0])
         sys.exit(1)
 
     # create the ratings data
@@ -162,8 +219,8 @@ if __name__ == '__main__':
             print('Usage: %s ratings filename' % sys.argv[0])
             sys.exit(1)
         _regenerate_ratings(sys.argv[2])
-    elif sys.argv[1] == 'auto-delete':
-        _auto_delete()
+    elif sys.argv[1] == 'fsck':
+        _fsck()
     elif sys.argv[1] == 'taboo-check':
         _taboo_check()
     elif sys.argv[1] == 'taboo-import':
diff --git a/app_data/httpd-pre-init/entrypoint.sh b/app_data/httpd-pre-init/entrypoint.sh
index cc48b42..501f702 100755
--- a/app_data/httpd-pre-init/entrypoint.sh
+++ b/app_data/httpd-pre-init/entrypoint.sh
@@ -3,4 +3,4 @@
 python36 /opt/app-root/src/wsgi-scripts/odrs-apache-setup.py ${HTTPD_CONFIGURATION_PATH}/odrs.gnome.org.conf
 
 while true; do python36 /opt/app-root/src/cron.py ratings /opt/app-root/src/odrs/static/ratings.json ; sleep 
43200 ; done &
-while true; do python36 /opt/app-root/src/cron.py auto-delete ; sleep 43200 ; done &
+while true; do python36 /opt/app-root/src/cron.py fsck ; sleep 43200 ; done &
diff --git a/app_data/migrations/versions/ef03b3a98056_.py b/app_data/migrations/versions/ef03b3a98056_.py
index 75c02a5..a5df523 100644
--- a/app_data/migrations/versions/ef03b3a98056_.py
+++ b/app_data/migrations/versions/ef03b3a98056_.py
@@ -22,20 +22,6 @@ def upgrade():
                         order_by(Component.app_id.asc()).all():
         components[component.app_id] = component
 
-    # guessed, thanks Canonical! :/
-    for app_id in components:
-        if not app_id.startswith('io.snapcraft.'):
-            continue
-        if components[app_id].component_id_parent:
-            continue
-        name, _ = app_id[13:].rsplit('-', maxsplit=1)
-        parent = components.get(name + '.desktop')
-        if not parent:
-            continue
-        print('adding snapcraft parent for {} -> {}'.format(components[app_id].app_id,
-                                                            parent.app_id))
-        parent.adopt(components[app_id])
-
     # from appstream-glib
     mapping = {
         'baobab.desktop': 'org.gnome.baobab.desktop',
@@ -155,19 +141,6 @@ def upgrade():
                                                          components[app_id_new].app_id))
         components[app_id_new].adopt(components[app_id])
 
-    # upstream drops the .desktop sometimes
-    for app_id in components:
-        if components[app_id].component_id_parent:
-            continue
-        app_id_new = app_id.replace('.desktop', '')
-        if app_id == app_id_new:
-            continue
-        if not app_id_new in components:
-            continue
-        print('adding parent for {} -> {}'.format(components[app_id].app_id,
-                                                  components[app_id_new].app_id))
-        components[app_id_new].adopt(components[app_id])
-
     # done
     db.session.commit()