[jhbuild/jhdebuild: 516/536] [jhdebuild] Add a DebianBasePackage object

[John Carr <johncarr src gnome org>]

commit 50cb91c3e5f5715d33c9e678fc58d6097f70a2a7
Author: Frederic Peters <fpeters 0d be>
Date:   Sun May 17 13:34:29 2009 +0100

    [jhdebuild] Add a DebianBasePackage object
---
 jhbuild/modtypes/__init__.py |  183 +++++++++++++++++++++++++++++++++++-
 jhbuild/modtypes/debian.py   |  214 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 393 insertions(+), 4 deletions(-)

diff --git a/jhbuild/modtypes/__init__.py b/jhbuild/modtypes/__init__.py
index 794cef3..b36ea22 100644
--- a/jhbuild/modtypes/__init__.py
+++ b/jhbuild/modtypes/__init__.py
@@ -27,10 +27,23 @@ __all__ = [
     'get_branch'
     ]
 
+try:
+    import apt_pkg
+except ImportError:
+    apt_pkg = None
+
 import os
+import re
 
 from jhbuild.errors import FatalError, CommandError, BuildStateError
 
+def lax_int(s):
+    try:
+        return int(s)
+    except ValueError:
+        return -1
+
+
 _module_types = {}
 def register_module_type(name, parse_func):
     _module_types[name] = parse_func
@@ -125,6 +138,8 @@ class SkipToState(Exception):
 class Package:
     type = 'base'
     STATE_START = 'start'
+    STATE_APT_GET_UPDATE = 'apt_get_update'
+    STATE_BUILD_DEPS     = 'build_deps'
     STATE_DONE  = 'done'
     def __init__(self, name, dependencies = [], after = [], suggests = [],
             extra_env = None):
@@ -144,6 +159,82 @@ class Package:
     def get_builddir(self, buildscript):
         raise NotImplementedError
 
+    def get_builddebdir(self, buildscript):
+        return os.path.normpath(os.path.join(self.get_builddir(buildscript), '..', 'debian'))
+
+    def get_debian_name(self, buildscript):
+        debian_name = buildscript.config.debian_names.get(self.name)
+        if not debian_name:
+            debian_name = self.name
+        return debian_name
+
+    def get_one_binary_package_name(self, buildscript):
+        debian_name = self.get_debian_name(buildscript)
+        sources = apt_pkg.GetPkgSrcRecords()
+        sources.Restart()
+        t = []
+        while sources.Lookup(debian_name):
+            try:
+                t.append((sources.Package, sources.Binaries, sources.Version))
+            except AttributeError:
+                pass
+        if not t:
+            raise KeyError
+        t.sort(lambda x, y: apt_pkg.VersionCompare(x[-1],y[-1]))
+        return t[-1][1][0]
+
+    def get_available_debian_version(self, buildscript):
+        apt_cache = apt_pkg.GetCache()
+        binary_name = self.get_one_binary_package_name(buildscript)
+        for pkg in apt_cache.Packages:
+            if pkg.Name == binary_name:
+                t = list(pkg.VersionList)
+                t.sort(lambda x, y: apt_pkg.VersionCompare(x.VerStr, y.VerStr))
+                return t[-1].VerStr
+        return None
+
+    def get_installed_debian_version(self):
+        apt_cache = apt_pkg.GetCache()
+        for pkg in apt_cache.Packages:
+            if pkg.Name == self.name:
+                return pkg.CurrentVer.VerStr
+        return None
+
+    def create_a_debian_dir(self, buildscript):
+        buildscript.set_action('Getting a debian/ directory for', self)
+        builddir = self.get_builddir(buildscript)
+        deb_sources = os.path.expanduser('~/.jhdebuild/apt-get-sources/')
+        if not os.path.exists(deb_sources):
+            os.makedirs(deb_sources)
+
+        debian_name = self.get_debian_name(buildscript)
+
+        try:
+            buildscript.execute(['apt-get', 'source', debian_name], cwd = deb_sources)
+        except CommandError:
+            raise BuildStateError('No debian source package for %s' % self.name)
+
+        dir = [x for x in os.listdir(deb_sources) if (
+                x.startswith(debian_name) and os.path.isdir(os.path.join(deb_sources, x)))][0]
+        buildscript.execute(['rm', '-rf', 'debian/*'], cwd = builddir)
+        if not os.path.exists(os.path.join(builddir, 'debian')):
+            os.mkdir(os.path.join(builddir, 'debian'))
+        buildscript.execute('cp -R %s/* debian/' % os.path.join(deb_sources, dir, 'debian'),
+                cwd = builddir)
+        file(os.path.join(builddir, 'debian', 'APPROPRIATE_FOR_JHDEBUILD'), 'w').write('')
+
+    def get_makefile_var(self, buildscript, variable_name):
+        builddir = self.get_builddir(buildscript)
+        makefile = os.path.join(builddir, 'Makefile')
+        if not os.path.exists(makefile):
+            return None
+        v = re.findall(r'\b%s *= *(.*)' % variable_name, open(makefile).read())
+        if v:
+            return v[0]
+        else:
+            return None
+
+
     def get_revision(self):
         return None
 
@@ -154,15 +245,23 @@ class Package:
         state or not.  If it returns True, go to do_$state.next_state and
         repeat.  If it returns False, return that state.
         """
+
+        if buildscript.config.debuild:
+            self.do_prefix = 'do_deb_'
+            self.skip_prefix = 'skip_deb_'
+        else:
+            self.do_prefix = 'do_'
+            self.skip_prefix = 'skip_'
+
         seen_states = []
-        state = getattr(self, 'do_' + last_state).next_state
+        state = getattr(self, self.do_prefix + last_state).next_state
         while True:
             seen_states.append(state)
             if state == self.STATE_DONE:
                 return state
-            do_method = getattr(self, 'do_' + state)
 
-            skip_method = getattr(self, 'skip_' + state)
+            do_method = getattr(self, self.do_prefix + state)
+            skip_method = getattr(self, self.skip_prefix + state)
             try:
                 if skip_method(buildscript, last_state):
                     state = do_method.next_state
@@ -180,7 +279,15 @@ class Package:
         Returns a tuple of the following form:
           (next-state, error-flag, [other-states])
         """
-        method = getattr(self, 'do_' + state)
+
+        if buildscript.config.debuild:
+            self.do_prefix = 'do_deb_'
+            self.skip_prefix = 'skip_deb_'
+        else:
+            self.do_prefix = 'do_'
+            self.skip_prefix = 'skip_'
+
+        method = getattr(self, self.do_prefix + state)
         try:
             method(buildscript)
         except SkipToState, e:
@@ -233,6 +340,68 @@ class Package:
                 raise SkipToState(self.STATE_DONE)
             return True
         return False
+    skip_deb_checkout = skip_checkout
+
+    def do_deb_start(self, buildscript):
+        buildscript.set_action('Starting building', self)
+        ext_dep = buildscript.config.external_dependencies.get(self.name)
+        if ext_dep:
+            available = self.get_available_debian_version(buildscript).split('-')[0]
+            if ':' in available: # remove epoch
+                available = available.split(':')[-1]
+
+            deb_available = [lax_int(x) for x in available.split('.')]
+            ext_minimum = [lax_int(x) for x in ext_dep.get('minimum').split('.')]
+            ext_recommended = [lax_int(x) for x in ext_dep.get('recommended').split('.')]
+
+            if deb_available >= ext_recommended:
+                buildscript.message('external dependency, available')
+                if not buildscript.config.build_external_deps == 'always':
+                    raise SkipToState(self.STATE_DONE)
+
+            if deb_available >= ext_minimum:
+                buildscript.message(
+                        'external dependency, available (but recommended version is not)')
+                if not buildscript.config.build_external_deps in ('always', 'recommended'):
+                    raise SkipToState(self.STATE_DONE)
+            else:
+                buildscript.message('external dependency, no version high enough')
+                if buildscript.config.build_external_deps == 'never':
+                    raise SkipToState(self.STATE_DONE)
+    do_deb_start.next_state = STATE_APT_GET_UPDATE
+    do_deb_start.error_states = []
+
+    def skip_deb_apt_get_update(self, buildscript, last_state):
+        return False
+
+    def do_deb_apt_get_update(self, buildscript):
+        if not buildscript.config.nonetwork:
+            buildscript.set_action('Updating packages database for', self)
+            try:
+                buildscript.execute(['sudo', 'apt-get', 'update'])
+            except CommandError:
+                pass
+    do_deb_apt_get_update.next_state = STATE_DONE
+    do_deb_apt_get_update.error_states = []
+
+    def skip_deb_build_deps(self, buildscript, last_state):
+        return False
+
+    def do_deb_build_deps(self, buildscript):
+        buildscript.set_action('Installing build deps for', self)
+        debian_name = self.get_debian_name(buildscript)
+        v = None
+        try:
+            v = self.get_available_debian_version(buildscript)
+        except KeyError:
+            pass
+        if v:
+            try:
+                buildscript.execute(['sudo', 'apt-get', '--yes', 'build-dep', debian_name])
+            except CommandError:
+                raise BuildStateError('Failed to install build deps')
+    do_deb_build_deps.next_state = STATE_DONE
+    do_deb_build_deps.error_states = []
 
 
 class MetaModule(Package):
@@ -250,6 +419,12 @@ class MetaModule(Package):
     do_start.next_state = Package.STATE_DONE
     do_start.error_states = []
 
+    def do_deb_start(self, buildscript):
+        pass
+    do_deb_start.next_state = Package.STATE_DONE
+    do_deb_start.error_states = []
+
+
 def parse_metamodule(node, config, url, repos, default_repo):
     id = node.getAttribute('id')
     dependencies, after, suggests = get_dependencies(node)
diff --git a/jhbuild/modtypes/debian.py b/jhbuild/modtypes/debian.py
new file mode 100644
index 0000000..0115182
--- /dev/null
+++ b/jhbuild/modtypes/debian.py
@@ -0,0 +1,214 @@
+import os
+import re
+import glob
+import apt_pkg
+
+from jhbuild.errors import FatalError, CommandError, BuildStateError
+
+from jhbuild.modtypes import Package, SkipToState
+from jhbuild.utils import debian
+
+class DebianBasePackage:
+
+    STATE_TAR_X          = 'tar_x'
+    STATE_DEBIAN_DIR     = 'debian_dir'
+    STATE_BUILD_PACKAGE  = 'build_package'
+    STATE_DINSTALL       = 'dinstall'
+    STATE_UPGRADE        = 'upgrade'
+
+    def do_deb_build_deps(self, buildscript):
+        if os.path.exists(self.get_tarball_dir(buildscript)):
+            buildscript.message('%s already has a tarball' % self.name)
+            next_state = self.STATE_TAR_X
+        else:
+            next_state = self.STATE_CONFIGURE
+        Package.do_deb_build_deps(self, buildscript)
+        raise SkipToState(next_state)
+
+
+    def skip_deb_debian_dir(self, buildscript, last_state):
+        return False
+
+    def do_deb_debian_dir(self, buildscript):
+        buildscript.set_action('Getting a debian/ directory for', self)
+
+        debian_name = self.get_debian_name(buildscript)
+
+        pkg_dirs = []
+        for pkg_dir in buildscript.config.debian_checkout_modules or []:
+            pkg_dirs.extend(glob.glob('%s/%s/*/%s' % (
+                buildscript.config.checkoutroot, pkg_dir, debian_name)))
+            pkg_dirs.extend(glob.glob('%s/%s/*/*/%s' % (
+                buildscript.config.checkoutroot, pkg_dir, debian_name)))
+        versions = []
+        for p in pkg_dirs:
+            if not buildscript.config.nonetwork:
+                buildscript.execute(['svn', 'update'], cwd = p)
+            chl = os.path.join(p, 'debian', 'changelog')
+            if not os.path.exists(chl):
+                continue
+            first_line = file(chl).readline()
+            version = re.findall(r'\((.*?)\)', first_line)
+            if version:
+                versions.append((p, version[0]))
+        versions.sort(lambda x, y: apt_pkg.VersionCompare(x[-1],y[-1]))
+
+        builddebdir = self.get_builddebdir(buildscript)
+        distdir = self.get_distdir(buildscript)
+
+        buildscript.execute(['rm', '-rf', os.path.join(distdir, 'debian')], cwd = builddebdir)
+
+        if versions:
+            dir = os.path.join(versions[-1][0], 'debian')
+            buildscript.execute(['cp', '-R', dir, distdir + '/'], cwd = builddebdir)
+            base_dir = os.path.join(builddebdir, distdir, 'debian')
+            for base, dirs, files in os.walk(base_dir, topdown = False):
+                if '.svn' in dirs:
+                    buildscript.execute(['rm', '-rf', os.path.join(base, '.svn')],
+                            cwd = base_dir)
+        else:
+            deb_sources = os.path.join(buildscript.config.checkoutroot, 'deb-src')
+            if not os.path.exists(deb_sources):
+                os.makedirs(deb_sources)
+
+            try:
+                buildscript.execute(['apt-get', 'source', debian_name], cwd = deb_sources)
+            except CommandError:
+                raise BuildStateError('No debian source package for %s' % self.name)
+
+            dir = [x for x in os.listdir(deb_sources) if (
+                    x.startswith(debian_name) and os.path.isdir(os.path.join(deb_sources, x)))][0]
+
+            buildscript.execute(['cp', '-R', os.path.join(deb_sources, dir, 'debian'), distdir + '/'],
+                    cwd = builddebdir)
+
+        debian_dir = os.path.join(builddebdir, distdir, 'debian')
+        if os.path.exists(os.path.join(debian_dir, 'patches')):
+            for patch_filename in ('libtoolize', 'reautogen', 'as-needed'):
+                for filename in os.listdir(os.path.join(debian_dir, 'patches')):
+                    if patch_filename in filename:
+                        buildscript.execute(['rm', '%s/debian/patches/%s' % (distdir, filename)],
+                                cwd = builddebdir)
+                        break
+                if os.path.exists(os.path.join(debian_dir, 'patches', 'series')):
+                    series = open(os.path.join(debian_dir, 'patches', 'series')).readlines()
+                    open(os.path.join(debian_dir, 'patches', 'series'), 'w').write(''.join(
+                            [x for x in series if not filename in x]))
+                if os.path.exists(os.path.join(debian_dir, 'patches', '00list')):
+                    series = open(os.path.join(debian_dir, 'patches', '00list')).readlines()
+                    open(os.path.join(debian_dir, 'patches', '00list'), 'w').write(''.join(
+                            [x for x in series if not filename in x]))
+
+        os.chmod(os.path.join(builddebdir, distdir, 'debian', 'rules'), 0755)
+    do_deb_debian_dir.next_state = STATE_BUILD_PACKAGE
+    do_deb_debian_dir.error_states = []
+
+    def skip_deb_build_package(self, buildscript, next_state):
+        builddebdir = self.get_builddebdir(buildscript)
+        changes_file = self.get_changes_file(buildscript)
+        if changes_file and os.path.exists(os.path.join(builddebdir, changes_file)):
+            return True
+
+        version = debian.get_version(buildscript, self.get_debian_name(buildscript))
+        if version == self.get_debian_version(buildscript):
+            return True
+
+        return False
+
+    def do_deb_build_package(self, buildscript):
+        buildscript.set_action('Building package', self)
+        builddebdir = self.get_builddebdir(buildscript)
+        debian_version = self.get_debian_version(buildscript)
+        builddebdir = os.path.join(self.get_builddebdir(buildscript), self.get_distdir(buildscript))
+        if debian_version not in open(os.path.join(builddebdir, 'debian', 'changelog')).readline():
+            buildscript.execute(['debchange', '--preserve', '-v', debian_version,
+                    '--distribution', 'UNRELEASED', 'jhdebuild snapshot'],
+                    cwd = builddebdir)
+
+        l = debian.check_build_depends(os.path.join(builddebdir, 'debian', 'control'))
+        if l:
+            # first phase is installing packages where there is no alternatives
+            command = ['sudo', 'apt-get', '--yes', 'install']
+            command.extend([x[0] for x in l if len(x) == 1])
+            try:
+                buildscript.execute(command)
+            except CommandError:
+                raise BuildStateError('failed to install build deps (%s)' % ', '.join(command[3:]))
+            l = debian.check_build_depends(os.path.join(builddebdir, 'debian', 'control'))
+            for ps in l:
+                for p in ps:
+                    try:
+                        buildscript.execute(['sudo', 'apt-get', '--yes', 'install', p])
+                    except CommandError:
+                        break
+                else:
+                    raise BuildStateError('failed to install build deps (%s)' % ' | '.join(ps))
+
+            l = debian.check_build_depends(os.path.join(builddebdir, 'debian', 'control'))
+            if l:
+                raise BuildStateError('failed to install build deps (%s)' % ', '.join(
+                        [' | '.join([y for y in x]) for x in l]))
+
+        buildscript.execute(['dpkg-buildpackage','-rfakeroot', '-us', '-uc', '-D'],
+                cwd = builddebdir)
+    do_deb_build_package.next_state = STATE_DINSTALL
+    do_deb_build_package.error_states = [STATE_DEBIAN_DIR]
+
+    def get_changes_file(self, buildscript):
+        debian_name = self.get_debian_name(buildscript)
+        deb_version = self.get_debian_version(buildscript)
+        if ':' in deb_version:
+            deb_version = deb_version.split(':')[-1] # remove epoch
+        builddebdir = self.get_builddebdir(buildscript)
+        changes_file = [x for x in os.listdir(builddebdir) if (
+                x.startswith('%s_%s' % (debian_name, deb_version)) and x.endswith('.changes'))]
+        if not changes_file:
+            return None
+        return changes_file[0]
+
+    def skip_deb_dinstall(self, buildscript, next_state):
+        version = debian.get_version(buildscript, self.get_debian_name(buildscript))
+        if version == self.get_debian_version(buildscript):
+            return True
+
+        return buildscript.config.nodinstall
+
+    def do_deb_dinstall(self, buildscript):
+        buildscript.set_action('Installing into repository', self)
+        builddebdir = self.get_builddebdir(buildscript)
+        changes_file = self.get_changes_file(buildscript)
+        if changes_file is None:
+            raise BuildStateError('no .changes file')
+        changes_file = os.path.join(builddebdir, changes_file)
+        debian.install_changes(buildscript, changes_file)
+
+        # packages have been installed in repository, remove them from here
+        in_files = False
+        files = [changes_file]
+        for line in open(changes_file).readlines():
+            if line.startswith('Files:'):
+                in_files = True
+                continue
+            if not in_files:
+                continue
+            if line and line[0] == ' ':
+                files.append(os.path.join(builddebdir, line.split()[-1]))
+        for f in files:
+            os.unlink(f)
+    do_deb_dinstall.next_state = STATE_UPGRADE
+    do_deb_dinstall.error_states = []
+
+    def skip_deb_upgrade(self, buildscript, last_state):
+        return False
+
+    def do_deb_upgrade(self, buildscript):
+        buildscript.set_action('Upgrading packages', self)
+        if not buildscript.config.nonetwork:
+            buildscript.execute(['sudo', 'apt-get', 'update'])
+            buildscript.execute(['sudo', 'apt-get', '--yes', 'upgrade'])
+    do_deb_upgrade.next_state = Package.STATE_DONE
+    do_deb_upgrade.error_states = []
+
+    def get_version(self, buildscript):
+        raise NotImplementedError
+