[ostree] ostbuild: Further work on regenerating a snapshot from a compose

From: Colin Walters <walters src gnome org>
To: commits-list gnome org
Cc:
Subject: [ostree] ostbuild: Further work on regenerating a snapshot from a compose
Date: Tue, 13 Mar 2012 14:42:11 +0000 (UTC)
commit 87e9a29f1101c3d10515ce9ac07223933111c390
Author: Colin Walters <walters verbum org>
Date:   Wed Mar 7 19:58:07 2012 -0500

    ostbuild: Further work on regenerating a snapshot from a compose

 Makefile-ostbuild.am                               |    1 +
 gnomeos/3.4/manifest.json                          |    6 +-
 src/ostbuild/pyostbuild/builtin_build.py           |   48 +++++++-------
 src/ostbuild/pyostbuild/builtin_checkout.py        |   12 ++--
 .../pyostbuild/builtin_chroot_compile_one.py       |   11 +--
 src/ostbuild/pyostbuild/builtin_gen_snapshot.py    |   68 ++++++++++++++++++++
 src/ostbuild/pyostbuild/builtin_resolve.py         |    7 ++-
 src/ostbuild/pyostbuild/main.py                    |    1 +
 8 files changed, 112 insertions(+), 42 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index b64aa07..f45eb0a 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -27,6 +27,7 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/builtin_chroot_compile_one.py	\
 	src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py	\
 	src/ostbuild/pyostbuild/builtin_compile_one.py	\
+	src/ostbuild/pyostbuild/builtin_gen_snapshot.py	\
 	src/ostbuild/pyostbuild/builtin_resolve.py	\
 	src/ostbuild/pyostbuild/builtin_status.py	\
 	src/ostbuild/pyostbuild/builtins.py		\
diff --git a/gnomeos/3.4/manifest.json b/gnomeos/3.4/manifest.json
index 4a3e8fb..214acc3 100644
--- a/gnomeos/3.4/manifest.json
+++ b/gnomeos/3.4/manifest.json
@@ -52,9 +52,6 @@
 
    		{"src": "gnome:linux-user-chroot"},
 
-		{"src": "gnome:ostree",
-		 "config-opts": ["--disable-documentation"]},
-
 		{"src": "gnome:libxml2"},
 
 		{"src": "gnome:libxslt"},
@@ -576,6 +573,9 @@
 		{"src": "gnome:libsoup",
 		 "config-args": ["--disable-tls-check"]},
 
+		{"src": "gnome:ostree",
+		 "config-opts": ["--disable-documentation"]},
+
 		{"src": "fd-gstreamer:gstreamer",
 		 "config-opts": ["--disable-tests",
 		                 "--disable-fatal-warnings"]},
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 7f1ca8d..aa7f272 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -100,12 +100,9 @@ class OstbuildBuild(builtins.Builtin):
         buildname = buildutil.manifest_buildname(self.manifest, meta)
         buildroot_name = buildutil.manifest_buildroot_name(self.manifest, meta)
 
-        (keytype, uri) = buildutil.parse_src_key(meta['src'])
-
-        mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
-        checkoutdir = os.path.join(self.workdir, 'src', name)
-        component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, branch,
-                                             overwrite=not self.args.debug_shell)
+        checkoutdir = os.path.join(self.workdir, 'src')
+        component_src = os.path.join(checkoutdir, name)
+        run_sync(['ostbuild', 'checkout', '--overwrite', '--manifest=' + self.manifest_path, name], cwd=checkoutdir)
 
         current_vcs_version = meta['revision']
 
@@ -116,10 +113,13 @@ class OstbuildBuild(builtins.Builtin):
         if previous_build_version is not None:
             log("Previous build of '%s' is %s" % (buildname, previous_build_version))
 
-            previous_vcs_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                        'show', '--print-metadata-key=ostbuild-revision',
-                                                        previous_build_version])
-            previous_vcs_version = previous_vcs_version.strip()
+            previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                                          'cat', previous_build_version,
+                                                          '/_ostbuild-meta.json'],
+                                                         log_initiation=True)
+            previous_meta = json.loads(previous_metadata_text)
+
+            previous_vcs_version = previous_meta['revision']
 
             vcs_version_matches = False
             if previous_vcs_version == current_vcs_version:
@@ -139,8 +139,6 @@ class OstbuildBuild(builtins.Builtin):
         json.dump(artifact_meta, f, indent=4, sort_keys=True)
         f.close()
 
-        run_sync(['ostbuild', 'checkout', '--manifest=' + self.manifest_path, name], cwd=checkoutdir)
-        
         logdir = os.path.join(self.workdir, 'logs', 'compile', name)
         old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name)
         if not os.path.isdir(logdir):
@@ -166,7 +164,6 @@ class OstbuildBuild(builtins.Builtin):
 
         args = ['ostree', '--repo=' + self.repo,
                 'commit', '-b', buildname, '-s', 'Build',
-                '--add-metadata-string=ostbuild-revision=' + artifact_meta['revision'],
                 '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
                 '--skip-if-unchanged']
 
@@ -188,33 +185,35 @@ class OstbuildBuild(builtins.Builtin):
         return True
 
     def _compose(self, components):
-        base_ref = self.manifest['base']
-
+        base = self.manifest['base']
+        base_branch = base['branch']
+        base_revision = base['revision']
         # HACK
         manifest_build_name = self.manifest['name']
         is_runtime = manifest_build_name.endswith('-runtime')
 
         branch_to_rev = {}
-        branches = [base_ref]
+        branch_to_subtrees = {}
+
+        component_branches = []
         for component in components:
             branch = buildutil.manifest_buildname(self.manifest, component)
-            branches.append(branch)
+            component_branches.append(branch)
 
         args = ['ostree', '--repo=' + self.repo,
                 'rev-parse']
-        args.extend(branches)
+        args.extend(component_branches)
         branch_revs_text = run_sync_get_output(args)
         branch_revs = branch_revs_text.split('\n')
 
-        for (branch, rev) in zip(branches, branch_revs):
+        for (branch, rev) in zip(component_branches, branch_revs):
             branch_to_rev[branch] = rev
 
-        branch_to_subtrees = {}
-        branch_to_subtrees[base_ref] = ['/']
-        contents = [base_ref]
+        contents = [base_branch]
+        branch_to_subtrees[base_branch] = ['/']
+        branch_to_rev[base_branch] = base_revision
         
-        for component in components:
-            branch = buildutil.manifest_buildname(self.manifest, component)
+        for branch in component_branches:
             contents.append(branch)
             subtrees = ['/runtime']
             branch_to_subtrees[branch] = subtrees
@@ -230,6 +229,7 @@ class OstbuildBuild(builtins.Builtin):
 
         metadata_contents = []
         metadata = {'source': 'ostbuild compose v0',
+                    'base': base,
                     'contents': metadata_contents}
         for branch in contents:
             branch_rev = branch_to_rev[branch]
diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py
index e77589e..d45a742 100755
--- a/src/ostbuild/pyostbuild/builtin_checkout.py
+++ b/src/ostbuild/pyostbuild/builtin_checkout.py
@@ -39,6 +39,7 @@ class OstbuildCheckout(builtins.Builtin):
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--manifest', required=True)
+        parser.add_argument('--overwrite', action='store_true')
         parser.add_argument('components', nargs='*')
 
         args = parser.parse_args(argv)
@@ -64,23 +65,22 @@ class OstbuildCheckout(builtins.Builtin):
 
             component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
                                                  component['revision'],
-                                                 overwrite=False)
+                                                 overwrite=args.overwrite)
 
             patches = component.get('patches')
             if patches is not None:
-                patches_meta = self.manifest['patches']
-                (patches_keytype, patches_uri) = buildutil.parse_src_key(patches_meta['src'])
+                (patches_keytype, patches_uri) = buildutil.parse_src_key(patches['src'])
                 patches_mirror = buildutil.get_mirrordir(self.mirrordir, patches_keytype, patches_uri)
                 vcs.get_vcs_checkout(self.mirrordir, patches_keytype, patches_uri,
-                                     self.patchdir, patches_meta['branch'],
+                                     self.patchdir, patches['branch'],
                                      overwrite=True)
 
-                patch_prefix = patches_meta.get('prefix', None)
+                patch_prefix = patches.get('prefix', None)
                 if patch_prefix is not None:
                     patchdir = os.path.join(self.patchdir, patch_prefix)
                 else:
                     patchdir = self.patchdir
-                for patch in patches:
+                for patch in patches['files']:
                     patch_path = os.path.join(patchdir, patch)
                     run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
         
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index fd0a35f..6569a39 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -35,7 +35,8 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         dependencies = components[:index]
 
         base = self.manifest['base']
-        checkout_trees = [(base, '/')]
+        base_revision = base['revision']
+        checkout_trees = [(base_revision, '/')]
         for dep in dependencies:
             buildname = buildutil.manifest_buildname(self.manifest, dep)
             checkout_trees.append((buildname, '/runtime'))
@@ -146,15 +147,9 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         env_copy['PWD'] = chroot_sourcedir
         run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
 
-        recorded_meta = dict(self.metadata)
-        del recorded_meta['revision']
-        patches_recorded_meta = recorded_meta.get('patches')
-        if patches_recorded_meta is not None:
-            del patches_recorded_meta['revision']
-
         recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json')
         recorded_meta_f = open(recorded_meta_path, 'w')
-        json.dump(recorded_meta, recorded_meta_f, indent=4, sort_keys=True)
+        json.dump(self.metadata, recorded_meta_f, indent=4, sort_keys=True)
         recorded_meta_f.close()
         
 builtins.register(OstbuildChrootCompileOne)
diff --git a/src/ostbuild/pyostbuild/builtin_gen_snapshot.py b/src/ostbuild/pyostbuild/builtin_gen_snapshot.py
new file mode 100755
index 0000000..c70b806
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_gen_snapshot.py
@@ -0,0 +1,68 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildGenSnapshot(builtins.Builtin):
+    name = "gen-snapshot"
+    short_description = "Generate a snapshot description from a tree"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('--branch', required=True)
+
+        args = parser.parse_args(argv)
+        self.args = args
+        self.parse_config()
+
+        contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                                  'cat', args.branch, 'contents.json'])
+        
+        contents = json.loads(contents_json_text)
+        contents_list = contents['contents']
+
+        base = contents_list[0]
+        artifacts = contents_list[1:]
+
+        components = []
+        snapshot = {'name': args.branch,
+                    'base': contents['base'],
+                    'components': components}
+
+        for artifact in artifacts:
+            component_meta_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                                       'cat', artifact['rev'], '/_ostbuild-meta.json'])
+            component_meta = json.loads(component_meta_text)
+            components.append(component_meta)
+
+        json.dump(snapshot, sys.stdout)
+    
+builtins.register(OstbuildGenSnapshot)
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index 5018b49..67fbd2c 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -240,7 +240,12 @@ class OstbuildResolve(builtins.Builtin):
                 del snapshot['base-prefix']
 
                 snapshot['name'] = '%s-%s-%s' % (name_prefix, architecture, component_type)
-                snapshot['base'] = '%s-%s-%s' % (base_prefix, architecture, component_type)
+
+                base_ref = '%s-%s-%s' % (base_prefix, architecture, component_type)
+                base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                                     'rev-parse', base_ref])
+                snapshot['base'] = {'branch': base_ref,
+                                    'revision': base_revision}
                 out_snapshot = os.path.join(self.workdir, snapshot['name'] + '.snapshot')
                 f = open(out_snapshot, 'w')
                 json.dump(snapshot, f, indent=4, sort_keys=True)
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 81b029c..b34d2e6 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -27,6 +27,7 @@ from . import builtin_checkout
 from . import builtin_chroot_compile_one
 from . import builtin_chroot_run_triggers
 from . import builtin_compile_one
+from . import builtin_gen_snapshot
 from . import builtin_resolve
 from . import builtin_status
[Date Prev][Date Next] [Thread Prev][Thread Next] [Thread Index] [Date Index] [Author Index]