[gimp-web/wip/Jehan/issue-236-new-sponsor-page: 11/21] content, plugins: improve the country verification for mirrors.

[Jehan <jehanp src gnome org>]

commit ba018c05b1db029aff5345a39135b54b527a6856
Author: Jehan <jehan girinstud io>
Date:   Sat Aug 28 17:39:38 2021 +0200

    content, plugins: improve the country verification for mirrors.
    
    - update GeoIP.dat as taken from the Debian package 'geoip-database':
      https://packages.debian.org/bullseye/all/geoip-database/download
      (version geoip-database_20191224-3_all.deb)
      In particular it fixes one wrong detection because we were using a
      very old version of IP database.
    - Remove GeoIP.dat.gz. Seems to come from old versions, we don't need
      this anymore.
    - Split the "location" field from mirrors.json into "country" which will
      contain the standard name of the country (i.e. it must be identical to
      the IP database naming) and "location" (containing additional
      information, such as city, or even more details depending on infos we
      have).
    - Our mirror list generation script won't break anymore on a mirror
      where resolution fails. We still print it with the country written in
      mirrors.json and output a warning.
    - When GeoIP localization and declared country are different, we now use
      the declared country and output a warning for verification.
    
    Basically we cannot trust 100% the IP database which might be wrong
    sometimes, but this is still a useful tool to verify declarations with
    this, for security (if a mismatch between declaration and reality
    happens, it may be an innocuous error or a server move which was simply
    not declared by mistake; but it may also be a security issue).
    
    We actually currently still have a mismatch (despite the DB update which
    did fix another mismatch but not this one): mirror.klaus-uwe.me is
    currently detected as being in Germany rather than in Austria. It might
    just be a DB mistake, but to be sure, I sent an email to the mirror
    administrator.

 content/downloads/mirrors.json       |  47 ++++++++++++++++++++++-------------
 plugins/gimp_mirrors/GeoIP.dat       | Bin 818578 -> 2099217 bytes
 plugins/gimp_mirrors/GeoIP.dat.gz    | Bin 470095 -> 0 bytes
 plugins/gimp_mirrors/gimp_mirrors.py |  24 ++++++++++++------
 4 files changed, 46 insertions(+), 25 deletions(-)
---
diff --git a/content/downloads/mirrors.json b/content/downloads/mirrors.json
index 66afbc56..0eaf21a6 100644
--- a/content/downloads/mirrors.json
+++ b/content/downloads/mirrors.json
@@ -1,6 +1,7 @@
 {
     "Artfiles New Media GmbH": {
-      "location": "Hamburg, Germany",
+      "location": "Hamburg",
+      "country": "Germany",
       "mirrors": [ "https://artfiles.org/gimp.org/pub/gimp/"; ],
       "about": "https://artfiles.org/";,
       "more": [
@@ -8,7 +9,7 @@
       ]
     },
     "Academic Computer Club, Umeå University": {
-      "location": "Sweden",
+      "country": "Sweden",
       "mirrors": [ "https://ftp.acc.umu.se/pub/gimp/gimp/"; ],
       "about": "https://ftp.acc.umu.se/";,
       "more": [
@@ -16,25 +17,28 @@
       ]
     },
     "University of Crete": {
-      "location": "Greece",
+      "country": "Greece",
       "mirrors": [ "https://ftp.cc.uoc.gr/mirrors/gimp/"; ],
       "about": "https://ftp.cc.uoc.gr/";,
       "more": [ ]
     },
     "Friedrich-Alexander-Universität Erlangen-Nürnberg (FAU)": {
-      "location": "Erlangen, Germany",
+      "location": "Erlangen",
+      "country": "Germany",
       "mirrors": [ "https://ftp.fau.de/gimp/gimp/"; ],
       "about": "https://ftp.fau.de/";,
       "more": [ ]
     },
     "Göttingen University and Max Planck Society": {
-      "location": "Göttingen, Germany",
+      "location": "Göttingen",
+      "country": "Germany",
       "mirrors": [ "https://ftp.gwdg.de/pub/misc/grafik/gimp/"; ],
       "about": "https://ftp.gwdg.de";,
       "more": [ ]
     },
     "Interdisciplinary Centre for Mathematical and Computational Modelling UW, University of Warsaw": {
-      "location": "Warsaw, Poland",
+      "location": "Warsaw",
+      "country": "Poland",
       "mirrors": [ "https://ftp.icm.edu.pl/pub/graphics/gimp/"; ],
       "about": "https://ftp.icm.edu.pl/";,
       "more": [
@@ -42,44 +46,49 @@
       ]
     },
     "Lysator ACS": {
-      "location": "Sweden",
+      "country": "Sweden",
       "mirrors": [ "https://ftp.lysator.liu.se/pub/gimp/"; ],
       "about": "https://www.lysator.liu.se/";,
       "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/125"; ]
     },
     "Studenten Net Twente (SNT), the University of Twente": {
-      "location": "Netherlands",
+      "country": "Netherlands",
       "mirrors": [ "https://ftp.snt.utwente.nl/pub/software/gimp/gimp/"; ],
       "about": "https://ftp.snt.utwente.nl/";,
       "more": [ ]
     },
     "IP-Connect": {
-      "location": "Vinnytsia, Ukraine",
+      "location": "Vinnytsia",
+      "country": "Ukraine",
       "mirrors": [ "https://gimp.ip-connect.info/gimp/"; ],
       "about": "https://ip-connect.info/";,
       "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/152"; ]
     },
     "Jaleco": {
-      "location": "Washington D.C., USA",
+      "location": "Washington D.C.",
+      "country": "United States",
       "mirrors": [ "https://mirror.jaleco.com/gimp/gimp/"; ],
       "about": "https://mirror.jaleco.com";,
       "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/63"; ]
     },
     "Klaus-Uwe Mitterer": {
-      "location": "Vienna, Austria",
+      "location": "Vienna",
+      "country": "Austria",
       "mirrors": [ "https://mirror.klaus-uwe.me/gimp/gimp/"; ],
       "about": "https://mirror.klaus-uwe.me/";,
       "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/31";,
                 "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/79"; ]
     },
     "Aalborg University": {
-      "location": "Aalborg, Denmark",
+      "location": "Aalborg",
+      "country": "Denmark",
       "mirrors": [ "https://mirrors.dotsrc.org/gimp/gimp/"; ],
       "about": "https://dotsrc.org/";,
       "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/68"; ]
     },
     "University of Maryland, College Park": {
-      "location": "College Park, MD, USA",
+      "location": "College Park, MD",
+      "country": "United States",
       "mirrors": [ "https://mirror.umd.edu/gimp/gimp/"; ],
       "about": "https://mirror.umd.edu/";,
       "more": [
@@ -87,7 +96,8 @@
       ]
     },
     "CSC - IT Center for Science / FUNET": {
-      "location": "Manchester, UK, Europe",
+      "location": "Espoo",
+      "country": "Finland",
       "mirrors": [ "https://www.nic.funet.fi/pub/mirrors/ftp.gimp.org/"; ],
       "about": "https://www.nic.funet.fi/README";,
       "more": [
@@ -95,7 +105,8 @@
       ]
     },
     "Korea FreeBSD Users Group": {
-      "location": "Seoul, South Korea, Asia",
+      "location": "Seoul",
+      "country": "Korea, Republic of",
       "mirrors": [ "https://ftp.kr.freebsd.org/pub/gimp/pub/gimp/"; ],
       "about": "https://www.kr.freebsd.org/mirrors/";,
       "more": [
@@ -103,7 +114,8 @@
       ]
     },
     "XMission": {
-      "location": "Salt Lake City, Utah, United States, North America",
+      "location": "Salt Lake City, Utah",
+      "country": "United States",
       "mirrors": [ "https://mirrors.xmission.com/gimp/gimp/"; ],
       "about": "https://mirrors.xmission.com/";,
       "more": [
@@ -111,7 +123,8 @@
       ]
     },
     "UKFast": {
-      "location": "Manchester, UK, Europe",
+      "location": "Manchester",
+      "country": "United Kingdom",
       "mirrors": [ "https://mirrors.ukfast.co.uk/sites/gimp.org/pub/gimp/"; ],
       "about": "https://mirrors.ukfast.co.uk/";,
       "more": [
diff --git a/plugins/gimp_mirrors/GeoIP.dat b/plugins/gimp_mirrors/GeoIP.dat
index eecf60b8..ca9612e6 100644
Binary files a/plugins/gimp_mirrors/GeoIP.dat and b/plugins/gimp_mirrors/GeoIP.dat differ
diff --git a/plugins/gimp_mirrors/gimp_mirrors.py b/plugins/gimp_mirrors/gimp_mirrors.py
index 6894b9bc..fca92ad3 100644
--- a/plugins/gimp_mirrors/gimp_mirrors.py
+++ b/plugins/gimp_mirrors/gimp_mirrors.py
@@ -72,18 +72,26 @@ def do_mirrors(path, context):
                 data = json.load(data_file)
 
                 for record in data:
+                    domain = urlparse(data[record]['mirrors'][0]).netloc
+                    country = data[record]['country']
                     try:
-                        domain = urlparse(data[record]['mirrors'][0]).netloc
                         country = gi.country_name_by_name(domain)
 
-                        if country in mirrors:
-                            mirrors[country][record] = data[record]['about']
-                        else:
-                            mirrors[country] = { record: data[record]['about'] }
-
+                        if country != data[record]['country']:
+                            print('WARNING: mismatch between GeoIP detection and declared mirror location:')
+                            print('         GeoIP detected country for "{}" ({}): "{}"'.format(record, 
domain, country))
+                            print('         Declared country for "{}" ({}): "{}"'.format(record, domain,
+                                                                                       
data[record]['country']))
+                            # If this happens, use the declared country,
+                            # yet still print a warning for verification.
+                            country = data[record]['country']
                     except:
-                        print("cannot resolve record: ", record)
+                        print("WARNING: cannot resolve record for {}: {} ".format(record, domain))
 
+                    if country in mirrors:
+                        mirrors[country][record] = data[record]['about']
+                    else:
+                        mirrors[country] = { record: data[record]['about'] }
                 try:
                     with open( path, 'rb') as f:
                         s = f.read()
@@ -97,7 +105,7 @@ def do_mirrors(path, context):
                         f.write(s.encode('utf-8'))
 
                 except:
-                    print("Trouble with reading/writing path.")
+                    print('ERROR: trouble with reading/writing "{}".'.format(path))
 
         except IOError:
             print("Cannot open /content/downloads/mirrors.json!")