[ostree/wip/ostbuild-v3] ostbuild: Tighten build process significantly
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/ostbuild-v3] ostbuild: Tighten build process significantly
- Date: Sun, 13 May 2012 19:59:02 +0000 (UTC)
commit 25a3b111a6fe59d130fd22e069add55c64549814
Author: Colin Walters <walters verbum org>
Date: Sun May 13 15:53:55 2012 -0400
ostbuild: Tighten build process significantly
First, "resolve" now just picks git commits. We don't expand
config-opts and patches, nor do we generate tree contents. This makes
the generated files *much* more human editable.
Next, fold "build-components" and "compose" into just "build". One
never really wants to just build components. This lets us eliminate
binary snapshots as a concept; instead we always have a combination of
source snapshot and component/ refs.
Makefile-ostbuild.am | 6 +-
...uiltin_build_components.py => builtin_build.py} | 293 +++++++++++---------
src/ostbuild/pyostbuild/builtin_checkout.py | 8 +-
.../pyostbuild/builtin_chroot_compile_one.py | 48 ++--
src/ostbuild/pyostbuild/builtin_compose.py | 113 --------
...iltin_tree_to_src.py => builtin_import_tree.py} | 8 +-
src/ostbuild/pyostbuild/builtin_modify_snapshot.py | 63 -----
src/ostbuild/pyostbuild/builtin_resolve.py | 115 +--------
src/ostbuild/pyostbuild/builtins.py | 37 ++-
src/ostbuild/pyostbuild/main.py | 5 +-
10 files changed, 215 insertions(+), 481 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 7a0f52b..dd1b2c7 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -24,20 +24,18 @@ pyostbuilddir=$(libdir)/ostbuild/pyostbuild
pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
src/ostbuild/pyostbuild/builtin_branch_prefix.py \
- src/ostbuild/pyostbuild/builtin_build_components.py \
+ src/ostbuild/pyostbuild/builtin_build.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
- src/ostbuild/pyostbuild/builtin_compose.py \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
src/ostbuild/pyostbuild/builtin_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_deploy_root.py \
+ src/ostbuild/pyostbuild/builtin_import_tree.py \
src/ostbuild/pyostbuild/builtin_pull_components.py \
src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_git_mirror.py \
src/ostbuild/pyostbuild/builtin_prefix.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
- src/ostbuild/pyostbuild/builtin_modify_snapshot.py \
- src/ostbuild/pyostbuild/builtin_tree_to_src.py \
src/ostbuild/pyostbuild/builtin_init.py \
src/ostbuild/pyostbuild/builtin_status.py \
src/ostbuild/pyostbuild/builtins.py \
diff --git a/src/ostbuild/pyostbuild/builtin_build_components.py b/src/ostbuild/pyostbuild/builtin_build.py
similarity index 53%
rename from src/ostbuild/pyostbuild/builtin_build_components.py
rename to src/ostbuild/pyostbuild/builtin_build.py
index b1b0102..082e474 100755
--- a/src/ostbuild/pyostbuild/builtin_build_components.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -37,9 +37,9 @@ from . import vcs
class BuildOptions(object):
pass
-class OstbuildBuildComponents(builtins.Builtin):
- name = "build-components"
- short_description = "Build multiple components from given source snapshot"
+class OstbuildBuild(builtins.Builtin):
+ name = "build"
+ short_description = "Build multiple components and generate trees"
def __init__(self):
builtins.Builtin.__init__(self)
@@ -62,19 +62,20 @@ class OstbuildBuildComponents(builtins.Builtin):
run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
fatal("Exiting after debug shell")
- def _build_one_component(self, basename, component, architecture):
+ def _build_one_component(self, component, architecture):
+ basename = component['name']
branch = component['branch']
- name = '%s/%s' % (basename, architecture)
- buildname = 'components/%s' % (name, )
+ archname = '%s/%s' % (basename, architecture)
+ buildname = 'components/%s' % (archname, )
current_vcs_version = component.get('revision')
+ expanded_component = self.expand_component(component)
+
# TODO - deduplicate this with chroot_compile_one
current_meta_io = StringIO()
- meta_copy = dict(component)
- meta_copy['name'] = basename # Note we have to match the name here
- json.dump(meta_copy, current_meta_io, indent=4, sort_keys=True)
+ json.dump(expanded_component, current_meta_io, indent=4, sort_keys=True)
current_metadata_text = current_meta_io.getvalue()
sha = hashlib.sha256()
sha.update(current_metadata_text)
@@ -90,7 +91,7 @@ class OstbuildBuildComponents(builtins.Builtin):
none_on_error=True)
if (current_vcs_version is not None
and previous_build_version is not None):
- log("Previous build of '%s' is %s" % (name, previous_build_version))
+ log("Previous build of '%s' is %s" % (archname, previous_build_version))
previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', previous_build_version,
@@ -102,33 +103,33 @@ class OstbuildBuildComponents(builtins.Builtin):
if current_meta_digest == previous_meta_digest:
log("Metadata is unchanged from previous")
- return False
+ return previous_build_version
else:
previous_metadata = json.loads(previous_metadata_text)
previous_vcs_version = previous_metadata['revision']
if current_vcs_version == previous_vcs_version:
log("Metadata differs; VCS version unchanged")
if self.buildopts.skip_vcs_matches:
- return False
- for k,v in meta_copy.iteritems():
+ return previous_build_version
+ for k,v in expanded_component.iteritems():
previous_v = previous_metadata.get(k)
if v != previous_v:
log("Key %r differs: old: %r new: %r" % (k, previous_v, v))
else:
log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
else:
- log("No previous build for '%s' found" % (name, ))
+ log("No previous build for '%s' found" % (archname, ))
checkoutdir = os.path.join(self.workdir, 'checkouts')
- fileutil.ensure_dir(checkoutdir)
- component_src = os.path.join(checkoutdir, basename)
+ component_src = os.path.join(checkoutdir, archname)
+ fileutil.ensure_parent_dir(component_src)
run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
'--checkoutdir=' + component_src,
'--clean', '--overwrite', basename])
artifact_meta = dict(component)
- logdir = os.path.join(self.workdir, 'logs', name)
+ logdir = os.path.join(self.workdir, 'logs', archname)
fileutil.ensure_dir(logdir)
log_path = os.path.join(logdir, 'compile.log')
if os.path.isfile(log_path):
@@ -139,7 +140,7 @@ class OstbuildBuildComponents(builtins.Builtin):
log("Logging to %s" % (log_path, ))
f = open(log_path, 'w')
chroot_args = self._get_ostbuild_chroot_args(architecture)
- chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
+ chroot_args.extend(['--name=' + basename, '--arch=' + architecture])
if self.buildopts.shell_on_failure:
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
if ecode != 0:
@@ -162,76 +163,81 @@ class OstbuildBuildComponents(builtins.Builtin):
f.close()
args.append('--statoverride=' + statoverride_path)
- component_resultdir = os.path.join(self.workdir, 'results', name)
+ component_resultdir = os.path.join(self.workdir, 'results', archname)
run_sync(args, cwd=component_resultdir)
if statoverride_path is not None:
os.unlink(statoverride_path)
- return True
-
- def _resolve_refs(self, refs):
- args = ['ostree', '--repo=' + self.repo, 'rev-parse']
- args.extend(refs)
- output = run_sync_get_output(args)
- return output.split('\n')
-
- def _save_bin_snapshot(self, components, component_architectures):
- bin_snapshot = dict(self.snapshot)
-
- del bin_snapshot['00ostree-src-snapshot-version']
- bin_snapshot['00ostree-bin-snapshot-version'] = 1
-
- for target in bin_snapshot['targets']:
- base = target['base']
- base_name = 'bases/%s' % (base['name'], )
- if 'ostree-revision' not in target:
- base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', base_name])
- base['ostree-revision'] = base_revision
-
- if 'architecture-buildroots2' in bin_snapshot:
- for arch,buildroot in bin_snapshot['architecture-buildroots2'].iteritems():
- name = buildroot['name']
- if 'ostree-revision' not in buildroot:
- rev = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', name])
- buildroot['ostree-revision'] = rev
-
- component_revisions = bin_snapshot.get('component-revisions', {})
-
- component_refs_to_resolve = []
- for name in components.iterkeys():
- for architecture in component_architectures[name]:
- archname = '%s/%s' % (name, architecture)
- if archname not in component_revisions:
- component_refs_to_resolve.append('components/' + archname)
-
- if len(component_refs_to_resolve) > 0:
- resolved_refs = self._resolve_refs(component_refs_to_resolve)
- for name,rev in zip(component_refs_to_resolve, resolved_refs):
- assert name.startswith('components/')
- archname = name[len('components/'):]
- component_revisions[archname] = rev
-
- bin_snapshot['component-revisions'] = component_revisions
-
- path = self.get_bin_snapshot_db().store(bin_snapshot)
- log("Binary snapshot: %s" % (path, ))
+
+ return run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', buildname])
+
+ def _compose_one_target(self, target, component_build_revs):
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+
+ compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
+ if os.path.isdir(compose_rootdir):
+ shutil.rmtree(compose_rootdir)
+ os.mkdir(compose_rootdir)
+
+ base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', base_name])
+
+ compose_contents = [(base_revision, '/')]
+ for tree_content in target['contents']:
+ name = tree_content['name']
+ rev = component_build_revs[name]
+ subtrees = tree_content['trees']
+ for subpath in subtrees:
+ compose_contents.append((rev, subpath))
+
+ (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ related_f = os.fdopen(related_fd, 'w')
+ contents_f = os.fdopen(contents_fd, 'w')
+ for (branch, subpath) in compose_contents:
+ related_f.write(' ')
+ related_f.write(branch)
+ related_f.write('\n')
+ contents_f.write(branch)
+ contents_f.write('\0')
+ contents_f.write(subpath)
+ contents_f.write('\0')
+ related_f.close()
+ contents_f.close()
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--user-mode', '--no-triggers', '--union',
+ '--from-file=' + contents_tmppath, compose_rootdir])
+ os.unlink(contents_tmppath)
+
+ contents_path = os.path.join(compose_rootdir, 'contents.json')
+ f = open(contents_path, 'w')
+ json.dump(bin_snapshot, f, indent=4, sort_keys=True)
+ f.close()
+
+ treename = 'trees/%s' % (target['name'], )
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'commit', '-b', treename, '-s', 'Compose',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--related-objects-file=' + related_tmppath,
+ '--skip-if-unchanged'], cwd=compose_rootdir)
+ os.unlink(related_tmppath)
+ shutil.rmtree(compose_rootdir)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--force-rebuild', action='store_true')
- parser.add_argument('--skip-vcs-matches', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
- parser.add_argument('--compose', action='store_true')
+ parser.add_argument('--force-rebuild', action='store_true')
+ parser.add_argument('--skip-vcs-matches', action='store_true')
+ parser.add_argument('--no-compose', action='store_true')
parser.add_argument('--compose-only', action='store_true')
- parser.add_argument('--start-at')
parser.add_argument('--shell-on-failure', action='store_true')
parser.add_argument('--debug-shell', action='store_true')
parser.add_argument('components', nargs='*')
-
-
args = parser.parse_args(argv)
self.args = args
@@ -239,15 +245,7 @@ class OstbuildBuildComponents(builtins.Builtin):
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
- component_revisions = self.snapshot.get('component-revisions', {})
-
- if component_revisions is not None:
- snapshot_type = "source+binary"
- else:
- snapshot_type = "source"
-
- log("Using %s snapshot: %s" % (snapshot_type,
- os.path.basename(self.snapshot_path), ))
+ log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
self.buildopts = BuildOptions()
self.buildopts.shell_on_failure = args.shell_on_failure
@@ -256,68 +254,87 @@ class OstbuildBuildComponents(builtins.Builtin):
self.force_build_components = set()
- required_components = {}
- component_architectures = {}
- for target in self.snapshot['targets']:
- for tree_content in target['contents']:
- (name, arch) = tree_content['name'].rsplit('/', 1)
- required_components[name] = self.snapshot['components'][name]
- if name not in component_architectures:
- component_architectures[name] = set([arch])
- else:
- component_architectures[name].add(arch)
+ components = self.snapshot['components']
- build_component_order = []
- if len(args.components) == 0:
- tsorted = buildutil.tsort_components(required_components, 'build-depends')
- tsorted.reverse()
- build_component_order = tsorted
- else:
- if args.start_at is not None:
- fatal("Can't specify --start-at with component list")
- for name in args.components:
- found = False
- component = self.snapshot['components'].get(name)
- if component is None:
- fatal("Unknown component %r" % (name, ))
- build_component_order.append(name)
- self.force_build_components.add(name)
-
- start_at_index = -1
- if args.start_at is not None:
- for i,component_name in enumerate(build_component_order):
- if component_name == args.start_at:
- start_at_index = i
- break
- if start_at_index == -1:
- fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
- else:
- start_at_index = 0
+ prefix = self.snapshot['prefix']
+ base_prefix = '%s/%s' % (self.snapshot['base']['name'], prefix)
+
+ architectures = self.snapshot['architectures']
+
+ component_to_arches = {}
+
+ runtime_components = []
+ devel_components = []
+
+ for component in components:
+ name = component['name']
+
+ is_runtime = component.get('component', 'runtime') == 'runtime'
+
+ if is_runtime:
+ runtime_components.append(component)
+ devel_components.append(component)
+
+ is_noarch = component.get('noarch', False)
+ if is_noarch:
+ # Just use the first specified architecture
+ component_arches = [architectures[0]]
+ else:
+ component_arches = component.get('architectures', architectures)
+ component_to_arches[name] = component_arches
+
+ for name in args.components:
+ component = components.get(name)
+ if component is None:
+ fatal("Unknown component %r" % (name, ))
+ self.force_build_components.add(name)
components_to_build = []
component_skipped_count = 0
+ component_build_revs = {}
+
if not args.compose_only:
- for component_name in build_component_order[start_at_index:]:
- component = required_components[component_name]
- architectures = component_architectures[component_name]
+ for component in components:
for architecture in architectures:
- archname = '%s/%s' % (component_name, architecture)
- if (component_revisions is not None and
- archname in component_revisions):
- component_skipped_count += 1
- else:
- components_to_build.append((component_name, component, architecture))
+ components_to_build.append((component, architecture))
log("%d components to build" % (len(components_to_build), ))
- if component_skipped_count > 0:
- log("%d components skipped due to existing component-revisions" % (component_skipped_count, ))
- for (component_name, component, architecture) in components_to_build:
- self._build_one_component(component_name, component, architecture)
+ for (component, architecture) in components_to_build:
+ archname = '%s/%s' % (component['name'], architecture)
+ build_rev = self._build_one_component(component, architecture)
+ component_build_revs[archname] = build_rev
+
+ targets_list = []
+ for target_component_type in ['runtime', 'devel']:
+ for architecture in architectures:
+ target = {}
+ targets_list.append(target)
+ target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type)
+
+ base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
+ target['base'] = {'name': base_ref}
+
+ if target_component_type == 'runtime':
+ target_components = runtime_components
+ else:
+ target_components = devel_components
+
+ contents = []
+ for component in target_components:
+ builds_for_component = component_to_arches[component['name']]
+ if architecture not in builds_for_component:
+ continue
+ binary_name = '%s/%s' % (component['name'], architecture)
+ component_ref = {'name': binary_name}
+ if target_component_type == 'runtime':
+ component_ref['trees'] = ['/runtime']
+ else:
+ component_ref['trees'] = ['/runtime', '/devel', '/doc']
+ contents.append(component_ref)
+ target['contents'] = contents
- self._save_bin_snapshot(required_components, component_architectures)
+ for target in targets_list:
+ self._compose_one_target(target, component_build_revs)
- if args.compose or args.compose_only:
- run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix])
-
-builtins.register(OstbuildBuildComponents)
+builtins.register(OstbuildBuild)
diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py
index e9c1142..9341548 100755
--- a/src/ostbuild/pyostbuild/builtin_checkout.py
+++ b/src/ostbuild/pyostbuild/builtin_checkout.py
@@ -60,7 +60,7 @@ class OstbuildCheckout(builtins.Builtin):
component_name = args.component
found = False
- component = self.get_component_meta(component_name)
+ component = self.get_expanded_component(component_name)
(keytype, uri) = buildutil.parse_src_key(component['src'])
is_local = (keytype == 'local')
@@ -82,9 +82,9 @@ class OstbuildCheckout(builtins.Builtin):
else:
checkoutdir = os.path.join(os.getcwd(), component_name)
fileutil.ensure_parent_dir(checkoutdir)
- vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
- component['revision'],
- overwrite=args.overwrite)
+ vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+ component['revision'],
+ overwrite=args.overwrite)
if args.clean:
if is_local:
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index 463fc93..9edab0c 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -34,6 +34,8 @@ class OstbuildChrootCompileOne(builtins.Builtin):
short_description = "Build artifacts from the current source directory in a chroot"
def _resolve_refs(self, refs):
+ if len(refs) == 0:
+ return []
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
args.extend(refs)
output = run_sync_get_output(args)
@@ -52,32 +54,27 @@ class OstbuildChrootCompileOne(builtins.Builtin):
shutil.rmtree(rootdir_tmp)
components = self.snapshot['components']
- dependencies = buildutil.build_depends(component_name, components)
- component = components.get(component_name)
+ component = None
+ build_dependencies = []
+ for component in components:
+ if component['name'] == component_name:
+ break
+ build_dependencies.append(component)
ref_to_rev = {}
- arch_buildroot_name = None
- arch_buildroot_rev = None
- if 'architecture-buildroots2' in self.snapshot:
- buildroots = self.snapshot['architecture-buildroots2']
- arch_buildroot = buildroots[architecture]
- arch_buildroot_name = arch_buildroot['name']
- arch_buildroot_rev = arch_buildroot.get('ostree-revision')
- else:
- buildroots = self.snapshot['architecture-buildroots']
- arch_rev_suffix = buildroots[architecture]
- arch_buildroot_name = 'bases/' + arch_rev_suffix
+ arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'],
+ self.snapshot['prefix'],
+ architecture)
- if arch_buildroot_rev is None:
- arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
- arch_buildroot_name]).strip()
+ arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
+ arch_buildroot_name]).strip()
ref_to_rev[arch_buildroot_name] = arch_buildroot_rev
checkout_trees = [(arch_buildroot_name, '/')]
refs_to_resolve = []
- for dependency_name in dependencies:
- buildname = 'components/%s/%s' % (dependency_name, architecture)
+ for dependency in build_dependencies:
+ buildname = 'components/%s/%s' % (dependency['name'], architecture)
refs_to_resolve.append(buildname)
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
@@ -152,7 +149,6 @@ class OstbuildChrootCompileOne(builtins.Builtin):
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--pristine', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--snapshot', required=True)
parser.add_argument('--name')
@@ -169,15 +165,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
else:
component_name = self.get_component_from_cwd()
- components = self.snapshot['components']
- component = components.get(component_name)
- if component is None:
- fatal("Couldn't find component '%s' in manifest" % (component_name, ))
- self.metadata = dict(component)
- self.metadata['name'] = component_name
- if not args.pristine:
- self.metadata['src'] = 'dirty:worktree'
- self.metadata['revision'] = 'dirty-worktree'
+ component = self.get_expanded_component(component_name)
workdir = self.workdir
@@ -202,7 +190,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
- json.dump(self.metadata, output_metadata, indent=4, sort_keys=True)
+ json.dump(component, output_metadata, indent=4, sort_keys=True)
output_metadata.close()
chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name)
@@ -229,7 +217,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json')
recorded_meta_f = open(recorded_meta_path, 'w')
- json.dump(self.metadata, recorded_meta_f, indent=4, sort_keys=True)
+ json.dump(component, recorded_meta_f, indent=4, sort_keys=True)
recorded_meta_f.close()
builtins.register(OstbuildChrootCompileOne)
diff --git a/src/ostbuild/pyostbuild/builtin_tree_to_src.py b/src/ostbuild/pyostbuild/builtin_import_tree.py
similarity index 92%
rename from src/ostbuild/pyostbuild/builtin_tree_to_src.py
rename to src/ostbuild/pyostbuild/builtin_import_tree.py
index b2df3b1..b24b0ae 100755
--- a/src/ostbuild/pyostbuild/builtin_tree_to_src.py
+++ b/src/ostbuild/pyostbuild/builtin_import_tree.py
@@ -28,9 +28,9 @@ from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
-class OstbuildTreeToSrc(builtins.Builtin):
- name = "tree-to-src"
- short_description = "Turn a tree into a source snapshot"
+class OstbuildImportTree(builtins.Builtin):
+ name = "import-tree"
+ short_description = "Extract source data from tree"
def __init__(self):
builtins.Builtin.__init__(self)
@@ -63,4 +63,4 @@ class OstbuildTreeToSrc(builtins.Builtin):
path = db.store(snapshot)
log("Source snapshot: %s" % (path, ))
-builtins.register(OstbuildTreeToSrc)
+builtins.register(OstbuildImportTree)
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index 721a580..35b2f60 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -35,7 +35,7 @@ from . import odict
class OstbuildResolve(builtins.Builtin):
name = "resolve"
- short_description = "Download the source code for a given manifest"
+ short_description = "Expand git revisions in source to exact targets"
def __init__(self):
builtins.Builtin.__init__(self)
@@ -88,8 +88,8 @@ class OstbuildResolve(builtins.Builtin):
self.prefix = self.manifest['prefix']
snapshot = copy.deepcopy(self.manifest)
- component_source_list = map(self._resolve_component_meta, self.manifest['components'])
- del snapshot['components']
+ components = map(self._resolve_component_meta, self.manifest['components'])
+ snapshot['components'] = components
if args.fetch:
if len(args.components) == 0:
@@ -112,6 +112,7 @@ class OstbuildResolve(builtins.Builtin):
fetch_components = []
global_patches_meta = self._resolve_component_meta(self.manifest['patches'])
+ snapshot['patches'] = global_patches_meta
(keytype, uri) = vcs.parse_src_key(global_patches_meta['src'])
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch'])
if args.fetch_patches:
@@ -120,7 +121,7 @@ class OstbuildResolve(builtins.Builtin):
global_patches_meta['revision'] = revision
unique_component_names = set()
- for component in component_source_list:
+ for component in components:
(keytype, uri) = vcs.parse_src_key(component['src'])
name = component['name']
@@ -133,112 +134,6 @@ class OstbuildResolve(builtins.Builtin):
component['branch'])
component['revision'] = revision
- config_opts = list(self.manifest['config-opts'])
- config_opts.extend(component.get('config-opts', []))
- component['config-opts'] = config_opts
-
- patch_files = component.get('patches')
- if patch_files is not None:
- component['patches'] = copy.deepcopy(global_patches_meta)
- component['patches']['files'] = patch_files
-
- manifest_architectures = snapshot['architectures']
-
- ostree_prefix = snapshot['prefix']
- base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix)
-
- snapshot['architecture-buildroots'] = {}
- for architecture in manifest_architectures:
- snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
- # Lame bit neeeded because I didn't have enough foresight to use an object
- # for this in the first place, and I don't want to break backwards compatibility
- # right now.
- snapshot['architecture-buildroots2'] = {}
- for architecture in manifest_architectures:
- snapshot['architecture-buildroots2'][architecture] = {'name': 'bases/%s-%s-devel' % (base_prefix, architecture)}
-
- components_by_name = {}
- component_ordering = []
- build_prev_component = None
- runtime_prev_component = None
- runtime_components = []
- devel_components = []
-
- builds = {}
-
- for component in component_source_list:
- base_name = component['name']
- name = '%s/%s' % (ostree_prefix, base_name)
- component['name'] = name
-
- components_by_name[name] = component
-
- if build_prev_component is not None:
- component['build-depends'] = [build_prev_component['name']]
- build_prev_component = component
-
- is_runtime = component.get('component', 'runtime') == 'runtime'
-
- if runtime_prev_component is not None:
- component['runtime-depends'] = [runtime_prev_component['name']]
-
- if is_runtime:
- runtime_prev_component = component
- runtime_components.append(component)
- devel_components.append(component)
-
- is_noarch = component.get('noarch', False)
- if is_noarch:
- # Just use the first specified architecture
- component_arches = [manifest_architectures[0]]
- else:
- component_arches = component.get('architectures', manifest_architectures)
- builds[name] = component_arches
-
- # We expanded these keys
- del snapshot['config-opts']
- del snapshot['vcsconfig']
- del snapshot['patches']
- del snapshot['architectures']
-
- targets_list = []
- snapshot['targets'] = targets_list
- for target_component_type in ['runtime', 'devel']:
- for architecture in manifest_architectures:
- target = {}
- targets_list.append(target)
- target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type)
-
- base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
- target['base'] = {'name': base_ref}
-
- if target_component_type == 'runtime':
- target_components = runtime_components
- else:
- target_components = devel_components
-
- contents = []
- for component in target_components:
- builds_for_component = builds[component['name']]
- if architecture not in builds_for_component:
- continue
- binary_name = '%s/%s' % (component['name'], architecture)
- component_ref = {'name': binary_name}
- if target_component_type == 'runtime':
- component_ref['trees'] = ['/runtime']
- else:
- component_ref['trees'] = ['/runtime', '/devel', '/doc']
- contents.append(component_ref)
- target['contents'] = contents
-
- for component in components_by_name.itervalues():
- del component['name']
- snapshot['components'] = components_by_name
-
- snapshot['00ostree-src-snapshot-version'] = 0
-
- current_time = time.time()
-
src_db = self.get_src_snapshot_db()
path = src_db.store(snapshot)
log("Source snapshot: %s" % (path, ))
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index 109be65..7d6dfb4 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -129,18 +129,30 @@ class Builtin(object):
'/_ostbuild-meta.json'])
return json.loads(text)
- def get_component_meta(self, name):
- assert self.repo is not None
+ def expand_component(self, component):
+ meta = dict(component)
+ global_patchmeta = self.snapshot.get('patches')
+ if global_patchmeta is not None:
+ component_patch_files = component.get('patches', [])
+ if len(component_patch_files) > 0:
+ patches = dict(global_patchmeta)
+ patches['files'] = component_patch_files
+ meta['patches'] = patches
+ config_opts = self.snapshot.get('config-opts', [])
+ config_opts.extend(component.get('config-opts', []))
+ meta['config-opts'] = config_opts
+ return meta
- if self.snapshot is not None:
- return self.snapshot['components'][name]
+ def get_component(self, name):
+ assert self.repo is not None
+ assert self.snapshot is not None
+ for component in self.snapshot['components']:
+ if component['name'] == name:
+ return component
+ fatal("Couldn't find component '%s' in manifest" % (component_name, ))
- meta = self._meta_cache.get(name)
- if meta is None:
- content = self.get_component_snapshot(name)
- meta = self.get_component_meta_from_revision(content['ostree-revision'])
- self._meta_cache[name] = meta
- return meta
+ def get_expanded_component(self, name):
+ return self.expand_component(self.get_component(name))
def get_prefix(self):
if self.prefix is None:
@@ -197,9 +209,10 @@ class Builtin(object):
else:
self.snapshot_path = path
self.snapshot = json.load(open(self.snapshot_path))
- src_ver = self.snapshot['00ostree-src-snapshot-version']
+ key = '00ostbuild-manifest-version'
+ src_ver = self.snapshot[key]
if src_ver != 0:
- fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0" % (src_ver, ))
+ fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, ))
def parse_bin_snapshot(self, prefix, path):
self.parse_prefix(prefix)
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 1c01bbc..a86cb95 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -22,21 +22,20 @@ import sys
import argparse
from . import builtins
-from . import builtin_build_components
+from . import builtin_build
from . import builtin_branch_prefix
from . import builtin_checkout
from . import builtin_chroot_compile_one
-from . import builtin_compose
from . import builtin_compile_one
from . import builtin_deploy_root
from . import builtin_deploy_qemu
+from . import builtin_import_tree
from . import builtin_git_mirror
from . import builtin_pull_components
from . import builtin_privhelper_deploy_qemu
from . import builtin_prefix
from . import builtin_resolve
from . import builtin_modify_snapshot
-from . import builtin_tree_to_src
from . import builtin_init
from . import builtin_status
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
