[jhbuild] jhbuild/utils/systeminstall.py: parallelized AptSystemInstall

[Christoph Reiter <creiter src gnome org>]

commit 6e3b35a10304a456fbddd33e2311b87fa5877e4c
Author: Karl-Philipp Richter <krichter722 aol de>
Date:   Thu Jul 13 23:43:44 2017 +0200

    jhbuild/utils/systeminstall.py: parallelized AptSystemInstall
    
    Searching apt packages through apt-file is slow.
    
    Conducting the search in different threads slightly speeds up this search.
    The functioning of the code remain untouched.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=784938

 jhbuild/utils/systeminstall.py | 111 +++++++++++++++++++++++++++--------------
 1 file changed, 73 insertions(+), 38 deletions(-)
---
diff --git a/jhbuild/utils/systeminstall.py b/jhbuild/utils/systeminstall.py
index 868b34ab..5db3e216 100644
--- a/jhbuild/utils/systeminstall.py
+++ b/jhbuild/utils/systeminstall.py
@@ -27,6 +27,7 @@ import imp
 import textwrap
 import time
 from StringIO import StringIO
+import re
 
 import cmds
 
@@ -411,44 +412,84 @@ class AptSystemInstall(SystemInstall):
     def __init__(self):
         SystemInstall.__init__(self)
 
-    def _get_package_for(self, filename, exact_match):
-        if exact_match:
-            proc = subprocess.Popen(['apt-file', '--fixed-string', 'search', filename],
-                                    stdout=subprocess.PIPE, close_fds=True)
-        else:
-            proc = subprocess.Popen(['apt-file', 'search', filename],
-                                    stdout=subprocess.PIPE, close_fds=True)
-        stdout = proc.communicate()[0]
-        if proc.returncode != 0:
-            return None
-        for line in StringIO(stdout):
+    def _apt_file_result_pkgconfig(self):
+        apt_file_result = subprocess.check_output(["apt-file", "search", "--regex", "\\.pc$"])
+        ret_value = []
+        for line in StringIO(apt_file_result):
             parts = line.split(':', 1)
             if len(parts) != 2:
                 continue
             name = parts[0]
-            path = parts[1]
-            # Ignore copies of the pkg-config files that are not from the
-            # libraries.
-            if '/lsb3' in path:
-                continue
-            if '/emscripten' in path:
+            path = parts[1].strip()
+            ret_value.append((name, path))
+        return ret_value
+
+    def _apt_file_result_regexp(self, paths):
+        regexp = self._build_apt_file_path_regexp(paths)
+        if regexp is None:
+            return []
+        apt_file_result = subprocess.check_output(["apt-file", "search", "--regex", regexp])
+        ret_value = []
+        for line in StringIO(apt_file_result):
+            parts = line.split(':', 1)
+            if len(parts) != 2:
                 continue
+            name = parts[0]
+            path = parts[1].strip()
+            ret_value.append((name, path))
+        return ret_value
 
-            # otherwise for now, just take the first match
-            return name
+    def _build_apt_file_path_regexp(self, paths):
+        if len(paths) == 0:
+            return None
+        first_path = paths[0][1]
+        ret_value = re.escape(first_path)
+        for modname, path in paths:
+            if path.startswith("/"):
+                path = path[1:]
+            ret_value += "|" + re.escape(path)
+        return ret_value
+
+    def _name_match_pkg(self, pkg, apt_file_result, native_packages):
+        for name, path in apt_file_result:
+            if path.endswith(pkg):
+                native_packages.append(name)
+                return True
+        return False
 
-    def _try_append_native_package(self, modname, filename, native_packages, exact_match):
-        native_pkg = self._get_package_for(filename, exact_match)
-        if native_pkg:
-            native_packages.append(native_pkg)
-            return True
+    def _name_match_exact(self, binary, apt_file_result, native_packages):
+        for name, path in apt_file_result:
+            if path == binary:
+                native_packages.append(name)
+                return True
         return False
 
-    def _append_native_package_or_warn(self, modname, filename, native_packages, exact_match):
-        if not self._try_append_native_package(modname, filename, native_packages, exact_match):
-            logging.info(_('No native package found for %(id)s '
-                           '(%(filename)s)') % {'id'       : modname,
-                                                'filename' : filename})
+    def _append_native_packages_or_warn_pkgconfig(self, pkgconfigs, native_packages):
+        apt_file_result = self._apt_file_result_pkgconfig()
+        for modname, pkg in pkgconfigs:
+            if not self._name_match_pkg(pkg, apt_file_result, native_packages):
+                logging.info(_('No native package found for %(id)s '
+                               '(%(filename)s)') % {'id'       : modname,
+                                                    'filename' : pkg})
+
+    def _append_native_packages_or_warn_exact(self, paths, native_packages):
+        apt_file_result = self._apt_file_result_regexp(paths)
+        for modname, path in paths:
+            if not self._name_match_exact(path, apt_file_result, native_packages):
+                logging.info(_('No native package found for %(id)s '
+                               '(%(filename)s)') % {'id'       : modname,
+                                                    'filename' : path})
+
+    def _append_native_packages_or_warn_c_includes(self, c_includes, native_packages, multiarch):
+        apt_file_result = self._apt_file_result_regexp(c_includes)
+        for modname, filename in c_includes:
+            # Try multiarch first, so we print the non-multiarch location on failure.
+            if (multiarch == None or
+                    not self._name_match_exact('/usr/include/%s/%s' % (multiarch, filename), 
apt_file_result, native_packages)):
+                if not self._name_match_exact('/usr/include/%s' % filename, apt_file_result, 
native_packages):
+                    logging.info(_('No native package found for %(id)s '
+                                   '(%(filename)s)') % {'id'       : modname,
+                                                        'filename' : filename})
 
     def _install_packages(self, native_packages, assume_yes):
         logging.info(_('Installing: %(pkgs)s') % {'pkgs': ' '.join(native_packages)})
@@ -466,13 +507,11 @@ class AptSystemInstall(SystemInstall):
 
         pkgconfigs = [(modname, '/%s.pc' % pkg) for modname, pkg in
                       get_uninstalled_pkgconfigs(uninstalled)]
-        for modname, filename in pkgconfigs:
-            self._append_native_package_or_warn(modname, filename, native_packages, False)
+        self._append_native_packages_or_warn_pkgconfig(pkgconfigs, native_packages)
 
         binaries = [(modname, '/usr/bin/%s' % pkg) for modname, pkg in
                     get_uninstalled_binaries(uninstalled)]
-        for modname, filename in binaries:
-            self._append_native_package_or_warn(modname, filename, native_packages, True)
+        self._append_native_packages_or_warn_exact(binaries, native_packages)
 
         # Get multiarch include directory, e.g. /usr/include/x86_64-linux-gnu
         multiarch = None
@@ -484,11 +523,7 @@ class AptSystemInstall(SystemInstall):
             multiarch = subprocess.check_output(['gcc', '-print-multiarch']).strip()
 
         c_includes = get_uninstalled_c_includes(uninstalled)
-        for modname, filename in c_includes:
-            # Try multiarch first, so we print the non-multiarch location on failure.
-            if (multiarch == None or
-                not self._try_append_native_package(modname, '/usr/include/%s/%s' % (multiarch, filename), 
native_packages, True)):
-                self._append_native_package_or_warn(modname, '/usr/include/%s' % filename, native_packages, 
True)
+        self._append_native_packages_or_warn_c_includes(c_includes, native_packages, multiarch)
 
         if native_packages:
             self._install_packages(native_packages, assume_yes=assume_yes)