[gnome-ostree/wip/integrated-build: 4/4] More submodule work

[Colin Walters <walters src gnome org>]

commit 19f2b497977c27a89a8b5a513aeebf6c6b0e5e16
Author: Colin Walters <walters verbum org>
Date:   Mon Sep 24 09:03:30 2012 -0400

    More submodule work

 Makefile-ostbuild.am                               |    7 +-
 src/ostbuild/pyostbuild/builtin_build.py           |   34 +++---
 src/ostbuild/pyostbuild/builtin_checkout.py        |  120 --------------------
 src/ostbuild/pyostbuild/builtin_git_mirror.py      |  105 -----------------
 src/ostbuild/pyostbuild/builtin_import_tree.py     |   96 ----------------
 src/ostbuild/pyostbuild/builtin_prefix.py          |   73 ------------
 ...{builtin_fetch.py => builtin_submodule_sync.py} |   51 ++++----
 src/ostbuild/pyostbuild/builtins.py                |    1 -
 src/ostbuild/pyostbuild/main.py                    |    7 +-
 9 files changed, 43 insertions(+), 451 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 1a27c07..d60e94d 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -36,20 +36,15 @@ pyostbuilddir=$(libdir)/ostbuild/pyostbuild
 pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/buildutil.py		\
 	src/ostbuild/pyostbuild/builtin_build.py	\
-	src/ostbuild/pyostbuild/builtin_checkout.py	\
 	src/ostbuild/pyostbuild/builtin_deploy_qemu.py	\
 	src/ostbuild/pyostbuild/builtin_deploy_root.py	\
-	src/ostbuild/pyostbuild/builtin_fetch.py	\
 	src/ostbuild/pyostbuild/builtin_run_qemu.py	\
-	src/ostbuild/pyostbuild/builtin_import_tree.py	\
 	src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py	\
 	src/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py	\
-	src/ostbuild/pyostbuild/builtin_git_mirror.py	\
-	src/ostbuild/pyostbuild/builtin_prefix.py	\
 	src/ostbuild/pyostbuild/builtin_repoweb_json.py	\
-	src/ostbuild/pyostbuild/builtin_resolve.py	\
 	src/ostbuild/pyostbuild/builtin_init.py	\
 	src/ostbuild/pyostbuild/builtin_source_diff.py	\
+	src/ostbuild/pyostbuild/builtin_submodule_sync.py	\
 	src/ostbuild/pyostbuild/builtins.py		\
 	src/ostbuild/pyostbuild/filemonitor.py		\
 	src/ostbuild/pyostbuild/fileutil.py		\
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 8928892..a33c319 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -178,8 +178,13 @@ class OstbuildBuild(builtins.Builtin):
     def _needs_rebuild(self, previous_metadata, new_metadata):
         build_keys = ['config-opts', 'src', 'revision']
         for k in build_keys:
-            if (k not in new_metadata) or (previous_metadata[k] != new_metadata[k]):
-                return 'key %r differs' % (k, )
+            if (k not in new_metadata):
+                return 'key %r removed from new_metadata' % (k, )
+            if k in previous_metadata:
+                oldval = previous_metadata[k]
+                newval = new_metadata[k]
+                if oldval != newval:
+                    return 'key %r differs (%r -> %r)' % (k, oldval, newval)
             
         if 'patches' in previous_metadata:
             if 'patches' not in new_metadata:
@@ -198,8 +203,8 @@ class OstbuildBuild(builtins.Builtin):
                 return 'patch sha256sums differ'
         return None
 
-    def _compute_sha256sums_for_patches(self, patchdir, component):
-        patches = buildutil.get_patch_paths_for_component(patchdir, component)
+    def _compute_sha256sums_for_patches(self, component):
+        patches = buildutil.get_patch_paths_for_component(self.srcdir, component)
         result = []
 
         for patch in patches:
@@ -217,9 +222,10 @@ class OstbuildBuild(builtins.Builtin):
         buildname = '%s/%s/%s' % (self.snapshot['prefix'], basename, architecture)
         build_ref = 'components/%s' % (buildname, )
 
-        current_vcs_version = component.get('revision')
+        expanded_component = component
+        expanded_component['revision'] = buildutil.get_git_version_describe(self._component_abspath(component))
 
-        expanded_component = self.expand_component(component)
+        current_vcs_version = expanded_component.get('revision')
 
         skip_rebuild = self.args.compose_only
 
@@ -244,18 +250,8 @@ class OstbuildBuild(builtins.Builtin):
                 fatal("--compose-only specified but no previous build of %s found" % (buildname, ))
 
         if 'patches' in expanded_component:
-            patches_revision = expanded_component['patches']['revision']
-            patchdir = os.path.join(self.srcdir, 'patches')
-            if ((previous_metadata is not None) and
-                'patches' in previous_metadata and
-                previous_metadata['patches']['revision'] == patches_revision):
-                # Copy over the sha256sums
-                expanded_component['patches'] = previous_metadata['patches']
-            else:
-                patches_sha256sums = self._compute_sha256sums_for_patches(patchdir, expanded_component)
-                expanded_component['patches']['files_sha256sums'] = patches_sha256sums
-        else:
-            patchdir = None
+            patches_sha256sums = self._compute_sha256sums_for_patches(expanded_component)
+            expanded_component['patches']['files_sha256sums'] = patches_sha256sums
 
         force_rebuild = (self.buildopts.force_rebuild or
                          basename in self.force_build_components)
@@ -331,6 +327,8 @@ class OstbuildBuild(builtins.Builtin):
                            '--ostbuild-meta=_ostbuild-meta.json'])
         env_copy = dict(buildutil.BUILD_ENV)
         env_copy['PWD'] = chroot_sourcedir
+        # Used to unbreak pulseaudio's git-version-gen
+        env_copy['GIT_DESCRIBE_FOR_BUILD'] = expanded_component['revision']
 
         log("Logging to %s" % (log_path, ))
         f = open(log_path, 'w')
diff --git a/src/ostbuild/pyostbuild/builtin_fetch.py b/src/ostbuild/pyostbuild/builtin_submodule_sync.py
similarity index 71%
rename from src/ostbuild/pyostbuild/builtin_fetch.py
rename to src/ostbuild/pyostbuild/builtin_submodule_sync.py
index 76bb9a8..389b37b 100755
--- a/src/ostbuild/pyostbuild/builtin_fetch.py
+++ b/src/ostbuild/pyostbuild/builtin_submodule_sync.py
@@ -32,34 +32,40 @@ from . import buildutil
 from . import kvfile
 from . import odict
 
-class OstbuildFetch(builtins.Builtin):
-    name = "fetch"
-    short_description = "Update components tracking git branches" 
+class OstbuildSubmoduleSync(builtins.Builtin):
+    name = "submodule-sync"
+    short_description = "Reset submodules to content from manifest" 
 
     def __init__(self):
         builtins.Builtin.__init__(self)
 
-    def _fetch_submodule(self, component):
-        run_sync(['git', 'fetch'], cwd=self._component_path(component))
-
     def _ensure_submodule(self, component):
         component_path = self._component_path(component)
         branch = component.get('branch')
         tag = component.get('tag')
-        if (branch is None) and (tag is not None):
-            branch = 'master' # Hack - git submodule bails trying to check out non-branches
         (keytype, uri) = vcs.parse_src_key(component['src'])
         if not os.path.isdir(component_path):
-            run_sync(['git', 'submodule', 'add', '-b', branch, uri, component_path])
+            run_sync(['git', 'submodule', 'add', uri, component_path])
 
     def _sync_submodule(self, component):
         component_path = self._component_path(component)
         branch = component.get('branch')
+        if branch:
+            branch = 'origin/' + branch
         tag = component.get('tag')
         tag_or_branch = tag or branch
         log("Syncing %s to %s" % (component['name'], tag_or_branch))
-        run_sync(['git', 'checkout', '-q', tag_or_branch], cwd=component_path,
+        run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=component_path, log_initiation=False)
+        run_sync(['git', 'checkout', '-f', '-q', tag_or_branch], cwd=component_path,
+                 log_initiation=False)
+        run_sync(['git', 'submodule', 'init'], cwd=component_path,
+                 log_initiation=False)
+        run_sync(['git', 'submodule', 'update', '--recursive'], cwd=component_path,
                  log_initiation=False)
+        if 'patches' in component:
+            for patch in component['patches']['files']:
+                patch_path = os.path.join(self.srcdir, 'patches', patch)
+                run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_path)
 
     def _clean_submodules(self):
         all_components = {self.snapshot['base']['name']: self.snapshot['base']}
@@ -78,32 +84,25 @@ class OstbuildFetch(builtins.Builtin):
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--keep-going', action='store_true',
-                            help="Don't exit on fetch failures")
-        parser.add_argument('components', nargs='*',
-                            help="List of component names to git fetch")
+        parser.add_argument('components', nargs='*')
 
         args = parser.parse_args(argv)
-        self.args = args
 
         self.parse_config()
         self.parse_snapshot()
 
+        if len(args.components) > 0:
+            components = args.components
+        else:
+            components = map(lambda x: x['name'], self.snapshot['components'])
+
         self._ensure_submodule(self.snapshot['base'])
         self._sync_submodule(self.snapshot['base'])
-        for component in self.snapshot['components']:
+        for component_name in components:
+            component = self.get_component(component_name)
             self._ensure_submodule(component)
             self._sync_submodule(component)
 
         self._clean_submodules()
-
-        if len(args.components) > 0:
-            fetch_components = args.components
-        else:
-            fetch_components = map(lambda x: x['name'], self.snapshot['components'])
-            fetch_components.append(self.snapshot['base']['name'])
-         
-        for component_name in fetch_components:
-            self._fetch_submodule(self.get_component(component_name))
         
-builtins.register(OstbuildFetch)
+builtins.register(OstbuildSubmoduleSync)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index ef1ae63..9492c70 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -168,7 +168,6 @@ class Builtin(object):
         base_meta = buildutil.resolve_component_meta(self.snapshot, self.snapshot['base'])
         self.snapshot['base'] = base_meta
 
-
     def parse_snapshot_from_current(self):
         if self.ostree_dir is None:
             fatal("/ostree directory not found")
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 0ab26e2..f3d9bcf 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -23,19 +23,14 @@ import argparse
 
 from . import builtins
 from . import builtin_build
-from . import builtin_checkout
 from . import builtin_deploy_root
 from . import builtin_deploy_qemu
-from . import builtin_fetch
-from . import builtin_git_mirror
-from . import builtin_import_tree
 from . import builtin_init
 from . import builtin_run_qemu
-from . import builtin_prefix
 from . import builtin_privhelper_deploy_qemu
 from . import builtin_privhelper_run_qemu
 from . import builtin_repoweb_json
-from . import builtin_resolve
+from . import builtin_submodule_sync
 from . import builtin_source_diff
 
 def usage(ecode):