GNOME.org
 

[jhbuild/wip/sysdeps: 3/3] Add new partial_build key, "sysdeps" command

commit b0e2763bcd1033cd7b4ec4c7ecf214d44118e4fb
Author: Colin Walters <walters verbum org>
Date:   Thu Jun 23 16:43:51 2011 -0400

    Add new partial_build key, "sysdeps" command
    
    Basically, when the partial_build key is set (which it is by default),
    when we're gathering a module list (for say "jhbuild build", or just
    "jhbuild list"), we look at the installed pkg-config files.  If there
    is a local module which matches the id of the tarball, we skip it.
    
    We should still be comparing versions.
    
    But, this allows me to skip building e.g. cairo and nspr if I have
    them on the system.
    
    To allow developers to fill in using the system if available, add a
    new "sysdeps" command.  This command parses the moduleset and finds
    all .pc files that are needed by tarballs from the given moduleset,
    and installs them.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=564373

 jhbuild/commands/Makefile.am   |    1 +
 jhbuild/commands/sysdeps.py    |   86 +++++++++++++++++++++++++++
 jhbuild/config.py              |    2 +-
 jhbuild/defaults.jhbuildrc     |    5 +-
 jhbuild/moduleset.py           |   38 +++++++++++-
 jhbuild/utils/Makefile.am      |    1 +
 jhbuild/utils/cmds.py          |   22 ++++---
 jhbuild/utils/systeminstall.py |  124 ++++++++++++++++++++++++++++++++++++++++
 8 files changed, 264 insertions(+), 15 deletions(-)
---
diff --git a/jhbuild/commands/Makefile.am b/jhbuild/commands/Makefile.am
index d5f3bc9..704fb7d 100644
--- a/jhbuild/commands/Makefile.am
+++ b/jhbuild/commands/Makefile.am
@@ -14,6 +14,7 @@ app_PYTHON = \
 	rdepends.py \
 	sanitycheck.py \
 	snapshot.py \
+	sysdeps.py \
 	tinderbox.py \
 	uninstall.py
 
diff --git a/jhbuild/commands/sysdeps.py b/jhbuild/commands/sysdeps.py
new file mode 100644
index 0000000..dd71fd0
--- /dev/null
+++ b/jhbuild/commands/sysdeps.py
@@ -0,0 +1,86 @@
+# jhbuild - a build script for GNOME 1.x and 2.x
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+#   sysdeps.py: Install system dependencies
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import os
+import sys
+import urllib
+from optparse import make_option
+import logging
+
+import jhbuild.moduleset
+import jhbuild.frontends
+from jhbuild.commands import Command, register_command
+import jhbuild.commands.base
+from jhbuild.commands.base import cmd_build
+from jhbuild.utils.cmds import check_version
+from jhbuild.utils.systeminstall import SystemInstall
+from jhbuild.versioncontrol.tarball import TarballBranch
+
+class cmd_sysdeps(cmd_build):
+    doc = N_('Check and install system dependencies')
+
+    name = 'sysdeps'
+
+    def __init__(self):
+        Command.__init__(self, [
+            make_option('--install',
+                        action='store_true', default = False,
+                        help=_('Install pkg-config modules via system'))])
+
+    def run(self, config, options, args, help=None):
+        config.set_from_cmdline_options(options)
+
+        installer = SystemInstall.find_best()
+        if installer is None:
+            raise FatalError(_("Don't know how to install packages on this system"))
+
+        module_set = jhbuild.moduleset.load(config)
+        modules = args or config.modules
+        module_list = module_set.get_module_list(modules, process_sysdeps=False)
+        module_state = module_set.get_system_modules(module_list)
+
+        for pkg_config,(module, req_version, installed_version, new_enough) in module_state.iteritems():
+            if (installed_version is not None) and new_enough:
+                print (_("pkg-config \"%(pkg)s\" is OK; required=%(req)s installed=%(installed)s" % {'pkg': pkg_config,
+                                                                                                     'req': req_version,
+                                                                                                     'installed': installed_version}))
+
+        print ""
+        for pkg_config,(module, req_version, installed_version, new_enough) in module_state.iteritems():
+            if (installed_version is not None) and (not new_enough):
+                print (_("pkg-config \"%(pkg)s\" installed on system is too old; required=%(req)s installed=%(installed)s" % {'pkg': pkg_config,
+                                                                                                                              'req': req_version,
+                                                                                                                              'installed': installed_version}))
+         
+        print ""
+        uninstalled = []
+        for pkg_config,(module, req_version, installed_version, new_enough) in module_state.iteritems():
+            if installed_version is None:
+                print (_("pkg-config \"%(pkg)s\" not installed; required=%(req)s") % {'pkg': pkg_config,
+                                                                                      'req': req_version})
+                uninstalled.append(pkg_config)
+
+        if options.install:
+            if len(uninstalled) == 0:
+                logging.info(_("No uninstalled system dependencies to install for modules: %r" % (modules, )))
+            else:
+                logging.info(_("Installing dependencies on system: %s" % (' '.join(uninstalled), )))
+            installer.install(uninstalled)
+
+register_command(cmd_sysdeps)
diff --git a/jhbuild/config.py b/jhbuild/config.py
index bcf5a9a..0488a16 100644
--- a/jhbuild/config.py
+++ b/jhbuild/config.py
@@ -40,7 +40,7 @@ _defaults_file = os.path.join(os.path.dirname(__file__), 'defaults.jhbuildrc')
 _default_jhbuildrc = os.path.join(os.environ['HOME'], '.jhbuildrc')
 
 _known_keys = [ 'moduleset', 'modules', 'skip', 'tags', 'prefix',
-                'checkoutroot', 'buildroot', 'top_builddir',
+                'partial_build', 'checkoutroot', 'buildroot', 'top_builddir',
                 'autogenargs', 'makeargs',
                 'installprog', 'repos', 'branches', 'noxvfb', 'xvfbargs',
                 'builddir_pattern', 'module_autogenargs', 'module_makeargs',
diff --git a/jhbuild/defaults.jhbuildrc b/jhbuild/defaults.jhbuildrc
index 9041028..9381d26 100644
--- a/jhbuild/defaults.jhbuildrc
+++ b/jhbuild/defaults.jhbuildrc
@@ -21,6 +21,9 @@ modules = [ 'meta-gnome-core', 'meta-gnome-apps-tested' ]
 #    have changed.
 build_policy = 'updated-deps'
 
+# If True, ignore tarball modules already installed while building
+partial_build = True
+
 # Skip modules installed more recently than the specified relative time.
 # min_age can only be specified via the command-line. Setting min_age within
 # the configuration file has no effect.
@@ -172,4 +175,4 @@ dvcs_mirror_dir = None
 
 # A string displayed before JHBuild executes a command. String may contain the
 # variables %(command)s, %(cwd)s
-print_command_pattern = '%(command)s'
\ No newline at end of file
+print_command_pattern = '%(command)s'
diff --git a/jhbuild/moduleset.py b/jhbuild/moduleset.py
index aacd4a9..51fc36b 100644
--- a/jhbuild/moduleset.py
+++ b/jhbuild/moduleset.py
@@ -36,8 +36,10 @@ except ImportError:
 from jhbuild import modtypes
 from jhbuild.versioncontrol import get_repo_type
 from jhbuild.utils import httpcache
-from jhbuild.utils.cmds import get_output
+from jhbuild.utils.cmds import compare_version, get_output
 from jhbuild.modtypes.testmodule import TestModule
+from jhbuild.versioncontrol.tarball import TarballBranch
+from jhbuild.utils.systeminstall import SystemInstall
 
 __all__ = ['load', 'load_tests', 'get_default_repo']
 
@@ -67,7 +69,7 @@ class ModuleSet:
 
     def get_module_list(self, seed, skip=[], tags=[], ignore_cycles=False,
                 ignore_suggests=False, include_optional_modules=False,
-                ignore_missing=False):
+                ignore_missing=False, process_sysdeps=True):
         '''gets a list of module objects (in correct dependency order)
         needed to build the modules in the seed list'''
 
@@ -99,6 +101,7 @@ class ModuleSet:
                                     'invalid': modname})
                     dep_missing = True
                     continue
+
                 if not depmod in all_modules:
                     all_modules.append(depmod)
 
@@ -125,6 +128,14 @@ class ModuleSet:
             # mark skipped modules as already processed
             self._state[self.modules.get(modname)] = 'processed'
 
+        # process_sysdeps lets us avoid repeatedly checking system module state when
+        # handling recursive dependencies.
+        if self.config.partial_build and process_sysdeps:
+            system_module_state = self.get_system_modules(all_modules)
+            for pkg_config,(module, req_version, installed_version, new_enough) in system_module_state.iteritems():
+                if new_enough:
+                    self._state[module] = 'processed'
+
         if tags:
             for modname in self.modules:
                 for tag in tags:
@@ -186,7 +197,7 @@ class ModuleSet:
                     # <http://bugzilla.gnome.org/show_bug.cgi?id=546640>
                     t_ms = ModuleSet(self.config)
                     t_ms.modules = self.modules.copy()
-                    dep_modules = t_ms.get_module_list(seed=[depmod.name])
+                    dep_modules = t_ms.get_module_list(seed=[depmod.name], process_sysdeps=False)
                     for m in dep_modules[:-1]:
                         if m in all_modules:
                             extra_afters.append(m)
@@ -228,6 +239,27 @@ class ModuleSet:
                 if test_app in mod.tested_pkgs:
                     test_modules.append(mod)
         return test_modules
+
+    def get_system_modules(self, modules):
+        if not self.config.partial_build:
+            logging.debug(_("Partial build is not enabled; can't get system modules"))
+            return ([], [])
+
+        installed_pkgconfig = SystemInstall.get_installed_pkgconfig()
+        
+        # pkgconfig -> (required_version, installed_verison)
+        module_state = {}
+        for module in modules:
+            if (module.pkg_config 
+                and isinstance(module.branch, TarballBranch)):
+                required_version = module.branch.version
+                if not module.pkg_config in installed_pkgconfig:
+                    module_state[module.pkg_config] = (module, required_version, None, False)
+                else:
+                    installed_version = installed_pkgconfig[module.pkg_config]
+                    new_enough = compare_version(installed_version, required_version)
+                    module_state[module.pkg_config] = (module, required_version, installed_version, new_enough)
+        return module_state
     
     def write_dot(self, modules=None, fp=sys.stdout, suggests=False, clusters=False):
         from jhbuild.modtypes import MetaModule
diff --git a/jhbuild/utils/Makefile.am b/jhbuild/utils/Makefile.am
index b6d63bf..1193ce1 100644
--- a/jhbuild/utils/Makefile.am
+++ b/jhbuild/utils/Makefile.am
@@ -7,6 +7,7 @@ app_PYTHON = \
 	notify.py \
 	packagedb.py \
 	sxml.py \
+	systeminstall.py \
 	trayicon.py \
 	unpack.py
 
diff --git a/jhbuild/utils/cmds.py b/jhbuild/utils/cmds.py
index 7a62f51..2aa2f5e 100644
--- a/jhbuild/utils/cmds.py
+++ b/jhbuild/utils/cmds.py
@@ -240,16 +240,7 @@ def has_command(cmd):
              return True
     return False
 
-def check_version(cmd, regexp, minver, extra_env=None):
-    try:
-        data = get_output(cmd, extra_env=extra_env)
-    except:
-        return False
-    match = re.match(regexp, data, re.MULTILINE)
-    if not match:
-        return False
-    version = match.group(1)
-
+def compare_version(version, minver):
     version = version.split('.')
     for i, ver in enumerate(version):
         part = re.sub(r'^[^\d]*(\d+).*$', r'\1', ver)
@@ -265,3 +256,14 @@ def check_version(cmd, regexp, minver, extra_env=None):
         else:
             minver[i] = int(part)
     return version >= minver
+
+def check_version(cmd, regexp, minver, extra_env=None):
+    try:
+        data = get_output(cmd, extra_env=extra_env)
+    except:
+        return False
+    match = re.match(regexp, data, re.MULTILINE)
+    if not match:
+        return False
+    version = match.group(1)
+    return compare_version(version, minver)
diff --git a/jhbuild/utils/systeminstall.py b/jhbuild/utils/systeminstall.py
new file mode 100644
index 0000000..c65859a
--- /dev/null
+++ b/jhbuild/utils/systeminstall.py
@@ -0,0 +1,124 @@
+# jhbuild - a build script for GNOME 1.x and 2.x
+# Copyright (C) 2011  Colin Walters <walters verbum org>
+#
+#   systeminstall.py - Use system-specific means to acquire dependencies
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import os
+import sys 
+import logging
+import subprocess
+from StringIO import StringIO
+
+import cmds
+
+_classes = []
+
+class SystemInstall(object):
+    def __init__(self):
+        pass
+
+    def install(self, pkgconfig_ids):
+        """Takes a list of pkg-config identifiers and uses a system-specific method to install them."""
+        raise NotImplementedError()
+
+    @classmethod
+    def get_installed_pkgconfig(cls):
+        """Returns a dictionary mapping pkg-config names to their current versions on the system."""
+        env = dict(os.environ)
+        if 'PKG_CONFIG_PATH' in env:
+            del env['PKG_CONFIG_PATH']
+        proc = subprocess.Popen(['pkg-config', '--list-all'], stdout=subprocess.PIPE, env=env, close_fds=True)
+        stdout = proc.communicate()[0]
+        proc.wait()
+        pkgs = []
+        for line in StringIO(stdout):
+            pkg, rest = line.split(None, 1)
+            pkgs.append(pkg + '.pc')
+        args = ['pkg-config', '--modversion']
+        args.extend(map(lambda x: x[:-3], pkgs))
+        proc = subprocess.Popen(args, stdout=subprocess.PIPE, close_fds=True)
+        stdout = proc.communicate()[0]
+        proc.wait()
+        pkgversions = {}
+        for pkg,verline in zip(pkgs, StringIO(stdout)):
+            pkgversions[pkg] = verline.strip()
+        return pkgversions
+
+    @classmethod
+    def find_best(cls):
+        for possible_cls in _classes:
+            if possible_cls.detect():
+                return possible_cls()
+
+class YumSystemInstall(SystemInstall):
+    def __init__(self):
+        SystemInstall.__init__(self)
+
+    def install(self, pkgconfig_ids):
+        # Explicitly qualify so we don't get the one in jhbuild root
+        args = ['/usr/bin/pkexec', 'yum', 'install']
+        pkgconfig_provides = map(lambda x: 'pkgconfig(' + x[:-3] + ')', pkgconfig_ids)
+        args.extend(pkgconfig_provides)
+        subprocess.check_call(args, close_fds=True)
+
+    @classmethod
+    def detect(cls):
+        return cmds.has_command('yum')
+
+_classes.append(YumSystemInstall)
+
+# NOTE: This class is unfinished
+class PkconSystemInstall(SystemInstall):
+    def __init__(self):
+        SystemInstall.__init__(self)
+
+    def _get_package_for(self, pkg_config):
+        assert pkg_config.endswith('.pc')
+        pkg_config = pkg_config[:-3]
+        proc = subprocess.Popen(['pkcon', '-p', 'what-provides', 'pkgconfig(%s)' % (pkg_config, ),
+                                 '--filter=arch;newest'], stdout=subprocess.PIPE, close_fds=True)
+        devnull.close()
+        stdout = proc.communicate()[0]
+        if proc.ecode != 0:
+            return None
+        pkg = None
+        for line in StringIO(stdout):
+            if line.startswith('Package:'):
+                pkg = line[line.find(':') + 1:].strip()
+                break
+        return pkg
+
+    def install(self, pkgconfig_ids):
+        required_pkgs = []
+        for pkg_config in pkgconfig_ids:
+            if not pkg.endswith('.pc'):
+                logging.warn("Invalid pkg-config id " + pkg)
+                continue
+            providing_pkg = self._get_package_for(pkg_config)
+            if providing_pkg is not None:
+                required_pkgs.append(providing_pkg)
+
+    @classmethod
+    def detect(cls):
+        return cmds.has_command('pkcon')
+
+_classes.append(PkconSystemInstall)
+
+if __name__ == '__main__':
+    installer = SystemInstall.find_best()
+    print "Using %r" % (installer, )
+    installer.install(sys.argv[1:])
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]