[gnome-ostree/wip/tasks-m1] Rewrite tasks system
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree/wip/tasks-m1] Rewrite tasks system
- Date: Mon, 4 Feb 2013 23:22:43 +0000 (UTC)
commit 3963e371f69b6db11ab47335a93325df3a1b080b
Author: Colin Walters <walters verbum org>
Date: Thu Jan 24 09:13:51 2013 -0500
Rewrite tasks system
Merge SubTask and DynTask for now - drop the dependencies, clean
things up a bit. Move resolve, build, builddisks to be tasks.
The autobuilder now is updated to use this.
Add a new bdiff task.
There's a new "ostbuild make" builtin that runs tasks. The benefit of
this when you do a build directly, like:
$ ostbuild make build/gnomeos-3.8
It's consistently run in a tempdir, logged etc., in the same way that
the autobuilder does it.
Makefile-ostbuild.am | 13 +-
src/ostbuild/js/buildutil.js | 70 ++---
src/ostbuild/js/builtins/autobuilder.js | 310 +++++-------------
src/ostbuild/js/builtins/git_mirror.js | 28 +-
src/ostbuild/js/builtins/make.js | 108 ++++++
src/ostbuild/js/builtins/resolve.js | 140 --------
src/ostbuild/js/builtins/run_task.js | 55 +++
src/ostbuild/js/dyntask.js | 244 --------------
src/ostbuild/js/jsonutil.js | 38 +++
src/ostbuild/js/main.js | 2 +
src/ostbuild/js/procutil.js | 2 +-
src/ostbuild/js/snapshot.js | 75 ++++-
src/ostbuild/js/subtask.js | 181 ----------
src/ostbuild/js/task.js | 345 ++++++++++++++++++++
src/ostbuild/js/tasks/task-bdiff.js | 153 +++++++++
.../js/{builtins/build.js => tasks/task-build.js} | 107 +++----
.../build_disks.js => tasks/task-builddisks.js} | 71 +++--
src/ostbuild/js/tasks/task-checksum.js | 123 -------
src/ostbuild/js/tasks/task-resolve.js | 84 +++++
src/ostbuild/js/vcs.js | 39 ++-
src/ostbuild/ostbuild.in | 2 +-
21 files changed, 1114 insertions(+), 1076 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index ef0ef1f..940aac4 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -43,7 +43,7 @@ jsostbuild_DATA= \
src/ostbuild/js/buildutil.js \
src/ostbuild/js/builtin.js \
src/ostbuild/js/config.js \
- src/ostbuild/js/dyntask.js \
+ src/ostbuild/js/task.js \
src/ostbuild/js/jsondb.js \
src/ostbuild/js/jsonutil.js \
src/ostbuild/js/main.js \
@@ -53,28 +53,29 @@ jsostbuild_DATA= \
src/ostbuild/js/procutil.js \
src/ostbuild/js/snapshot.js \
src/ostbuild/js/streamutil.js \
- src/ostbuild/js/subtask.js \
src/ostbuild/js/vcs.js \
$(NULL)
jsostbuiltinsdir=$(jsostbuilddir)/builtins
jsostbuiltins_DATA= \
src/ostbuild/js/builtins/autobuilder.js \
- src/ostbuild/js/builtins/build.js \
- src/ostbuild/js/builtins/build_disks.js \
src/ostbuild/js/builtins/checkout.js \
src/ostbuild/js/builtins/git_mirror.js \
+ src/ostbuild/js/builtins/make.js \
src/ostbuild/js/builtins/qa_make_disk.js \
src/ostbuild/js/builtins/qa_pull_deploy.js \
src/ostbuild/js/builtins/qa_smoketest.js \
src/ostbuild/js/builtins/prefix.js \
- src/ostbuild/js/builtins/resolve.js \
+ src/ostbuild/js/builtins/run_task.js \
src/ostbuild/js/builtins/shell.js \
$(NULL)
jsosttasksdir=$(jsostbuilddir)/tasks
jsosttasks_DATA= \
- src/ostbuild/js/tasks/task-checksum.js \
+ src/ostbuild/js/tasks/task-build.js \
+ src/ostbuild/js/tasks/task-resolve.js \
+ src/ostbuild/js/tasks/task-bdiff.js \
+ src/ostbuild/js/tasks/task-builddisks.js \
$(NULL)
endif
diff --git a/src/ostbuild/js/buildutil.js b/src/ostbuild/js/buildutil.js
index 0f3d1c7..a085992 100644
--- a/src/ostbuild/js/buildutil.js
+++ b/src/ostbuild/js/buildutil.js
@@ -42,50 +42,7 @@ function parseSrcKey(srckey) {
return [keytype, uri];
}
-function resolveComponent(manifest, componentMeta) {
- let result = {};
- Lang.copyProperties(componentMeta, result);
- let origSrc = componentMeta['src'];
- let didExpand = false;
- let vcsConfig = manifest['vcsconfig'];
- for (let vcsprefix in vcsConfig) {
- let expansion = vcsConfig[vcsprefix];
- let prefix = vcsprefix + ':';
- if (origSrc.indexOf(prefix) == 0) {
- result['src'] = expansion + origSrc.substr(prefix.length);
- didExpand = true;
- break;
- }
- }
-
- let name = componentMeta['name'];
- let src, idx, name;
- if (name == undefined) {
- if (didExpand) {
- src = origSrc;
- idx = src.lastIndexOf(':');
- name = src.substr(idx+1);
- } else {
- src = result['src'];
- idx = src.lastIndexOf('/');
- name = src.substr(idx+1);
- }
- let i = name.lastIndexOf('.git');
- if (i != -1 && i == name.length - 4) {
- name = name.substr(0, name.length - 4);
- }
- name = name.replace(/\//g, '-');
- result['name'] = name;
- }
-
- let branchOrTag = result['branch'] || result['tag'];
- if (!branchOrTag) {
- result['branch'] = 'master';
- }
-
- return result;
-}
function getPatchPathsForComponent(patchdir, component) {
let patches = component['patches'];
@@ -127,3 +84,30 @@ function getBaseUserChrootArgs() {
let path = findUserChrootPath();
return [path.get_path(), '--unshare-pid', '--unshare-ipc', '--unshare-net'];
}
+
+function compareVersions(a, b) {
+ let adot = a.indexOf('.');
+ while (adot != -1) {
+ let bdot = b.indexOf('.');
+ if (bdot == -1)
+ return 1;
+ let aSub = parseInt(a.substr(0, adot));
+ let bSub = parseInt(b.substr(0, bdot));
+ if (aSub > bSub)
+ return 1;
+ else if (aSub < bSub)
+ return -1;
+ a = a.substr(adot + 1);
+ b = b.substr(bdot + 1);
+ adot = a.indexOf('.');
+ }
+ if (b.indexOf('.') != -1)
+ return -1;
+ let aSub = parseInt(a);
+ let bSub = parseInt(b);
+ if (aSub > bSub)
+ return 1;
+ else if (aSub < bSub)
+ return -1;
+ return 0;
+}
diff --git a/src/ostbuild/js/builtins/autobuilder.js b/src/ostbuild/js/builtins/autobuilder.js
index 17268c8..18709fb 100644
--- a/src/ostbuild/js/builtins/autobuilder.js
+++ b/src/ostbuild/js/builtins/autobuilder.js
@@ -23,7 +23,7 @@ const Format = imports.format;
const GSystem = imports.gi.GSystem;
const Builtin = imports.builtin;
-const SubTask = imports.subtask;
+const Task = imports.task;
const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const JsonUtil = imports.jsonutil;
@@ -54,14 +54,12 @@ const Autobuilder = new Lang.Class({
this._stages = ['resolve', 'build', 'builddisks', 'smoke'];
- this._build_needed = true;
- this._do_builddisks = false;
- this._do_qa = false;
- this._full_resolve_needed = true;
- this._queued_force_resolve = [];
- this._resolve_timeout = 0;
- this._source_snapshot_path = null;
- this._prev_source_snapshot_path = null;
+ this._buildNeeded = true;
+ this._fullResolveNeeded = true;
+ this._resolveNeeded = false;
+ this._resolveTimeout = 0;
+ this._sourceSnapshotPath = null;
+ this._prevSourceSnapshotPath = null;
},
execute: function(args, loop, cancellable) {
@@ -69,13 +67,16 @@ const Autobuilder = new Lang.Class({
this._autoupdate_self = args.autoupdate_self;
if (!args.stage)
- args.stage = 'smoke';
+ args.stage = 'build';
this._stageIndex = this._stages.indexOf(args.stage);
if (this._stageIndex < 0)
throw new Error("Unknown stage " + args.stage);
this._do_builddisks = this._stageIndex >= this._stages.indexOf('builddisks');
this._do_smoke = this._stageIndex >= this._stages.indexOf('smoke');
+ this._resolveTaskName = 'resolve/' + this.prefix;
+ this._buildTaskName = 'build/' + this.prefix;
+
this._status_path = this.workdir.get_child('autobuilder-' + this.prefix + '.json');
this._manifestPath = Gio.File.new_for_path('manifest.json');
@@ -89,35 +90,43 @@ const Autobuilder = new Lang.Class({
this._snapshot_dir = this.workdir.get_child('snapshots').get_child(this.prefix);
this._src_db = new JsonDB.JsonDB(this._snapshot_dir);
- let taskdir = this.workdir.get_child('tasks');
- this._resolve_taskset = new SubTask.TaskSet(taskdir.get_child(this.prefix + '-resolve'));
- this._build_taskset = new SubTask.TaskSet(taskdir.get_child(this.prefix + '-build'));
- this._builddisks_taskset = new SubTask.TaskSet(taskdir.get_child(this.prefix + '-build-disks'));
- this._smoke_taskset = new SubTask.TaskSet(taskdir.get_child(this.prefix + '-smoke'));
-
- this._source_snapshot_path = this._src_db.getLatestPath();
+ this._taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
+ { onEmpty: Lang.bind(this, this._onTasksComplete) });
+ this._taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
+ this._sourceSnapshotPath = this._src_db.getLatestPath();
this._status_path = this.workdir.get_child('autobuilder-' + this.prefix + '.json');
- this._resolve_timeout = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT,
- 60 * 10, Lang.bind(this, this._fetchAll));
- this._fetchAll();
- if (this._source_snapshot_path != null)
- this._run_build();
+ this._resolveTimeout = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT,
+ 60 * 10, Lang.bind(this, this._triggerFullResolve));
+ this._runResolve();
+ if (this._sourceSnapshotPath != null)
+ this._runBuild();
this._updateStatus();
loop.run();
},
+ _onTasksComplete: function() {
+ },
+
+ _onTaskCompleted: function(taskmaster, task, success, error) {
+ if (task.name == this._resolveTaskName) {
+ this._onResolveExited(task, success, error);
+ } else if (task.name == this._buildTaskName) {
+ this._onBuildExited(task, success, error);
+ }
+ this._updateStatus();
+ },
+
_updateStatus: function() {
let newStatus = "";
- if (this._resolve_taskset.isRunning())
- newStatus += "[resolving] ";
- if (this._build_taskset.isRunning())
- newStatus += " [building] ";
- if (this._builddisks_taskset.isRunning())
- newStatus += " [disks] ";
+ let taskstateList = this._taskmaster.getTaskState();
+ for (let i = 0; i < taskstateList.length; i++) {
+ let taskstate = taskstateList[i];
+ newStatus += (taskstate.task.name + " ");
+ }
if (newStatus == "")
newStatus = "[idle]";
if (newStatus != this._status) {
@@ -125,8 +134,6 @@ const Autobuilder = new Lang.Class({
print(this._status);
this._impl.emit_property_changed('Status', new GLib.Variant("s", this._status));
}
-
- this._writeStatusFile();
},
get Status() {
@@ -135,234 +142,89 @@ const Autobuilder = new Lang.Class({
queueResolve: function(srcUrls) {
let matchingComponents = [];
- let snapshotData = this._src_db.loadFromPath(this._source_snapshot_path, null);
- let snapshot = new Snapshot.Snapshot(snapshotData, this._source_snapshot_path);
+ let snapshotData = this._src_db.loadFromPath(this._sourceSnapshotPath, null);
+ let snapshot = new Snapshot.Snapshot(snapshotData, this._sourceSnapshotPath);
+ let matched = false;
for (let i = 0; i < srcUrls.length; i++) {
let matches = snapshot.getMatchingSrc(srcUrls[i]);
- for (let j = 0; j < matches.length; j++)
- matchingComponents.push(matches[j]['name']);
- }
- if (matchingComponents.length > 0) {
- this._queued_force_resolve.push.apply(this._queued_force_resolve, matchingComponents);
- print("queued resolves: " + matchingComponents.join(' '));
- if (!this._resolve_taskset.isRunning())
- this._fetch();
- } else {
- print("Ignored fetch requests for unknown URLs: " + srcUrls.join(','));
+ for (let j = 0; j < matches.length; j++) {
+ this._queuedForceResolve.push.apply(this._queuedForceResolve, matches[i]['name']);
+ matched = true;
+ }
}
+ if (matched)
+ this._resolveNeeded = true;
+ this._runResolve();
},
- _fetchAll: function() {
- this._full_resolve_needed = true;
- if (!this._resolve_taskset.isRunning())
- this._fetch();
+ _triggerFullResolve: function() {
+ this._fullResolveNeeded = true;
+ this._runResolve();
return true;
},
- _fetch: function() {
+ _runResolve: function() {
let cancellable = null;
+
+ if (!(this._resolveNeeded || this._fullResolveNeeded))
+ return;
+
+ if (this._taskmaster.isTaskQueued(this._resolveTaskName))
+ return;
if (this._autoupdate_self)
ProcUtil.runSync(['git', 'pull', '-r'], cancellable)
- let args = ['ostbuild', 'resolve', '--manifest=manifest.json',
- '--fetch', '--fetch-keep-going'];
- let isFull;
- if (this._full_resolve_needed) {
- this._full_resolve_needed = false;
- isFull = true;
- } else if (this._queued_force_resolve.length > 0) {
- args.push.apply(args, this._queued_force_resolve);
- isFull = false;
+ if (this._fullResolveNeeded) {
+ this._fullResolveNeeded = false;
+ this._taskmaster.pushTask(this._resolveTaskName,
+ { fetchAll: true });
} else {
- throw new Error("_fetch() when not needed");
+ this._taskmaster.pushTask(this._resolveTaskName,
+ { fetchComponents: this._queuedForceResolve });
}
- this._queued_force_resolve = [];
- let context = new GSystem.SubprocessContext({ argv: args });
- let workdir = this._resolve_taskset.prepare();
- let tmpManifest = workdir.get_child(this._manifestPath.get_basename());
- GSystem.file_linkcopy(this._manifestPath, tmpManifest, Gio.FileCopyFlags.OVERWRITE, cancellable);
- let t = this._resolve_taskset.start(context,
- cancellable,
- Lang.bind(this, this._onResolveExited));
- print(Format.vprintf("Resolve task %s started (%s)", [t.versionstr, isFull ? "full" : "incremental"]));
+ this._queuedForceResolve = [];
this._updateStatus();
-
- return false;
},
_onResolveExited: function(resolveTask, success, msg) {
print(Format.vprintf("resolve exited; success=%s msg=%s", [success, msg]))
- this._prev_source_snapshot_path = this._source_snapshot_path;
- this._source_snapshot_path = this._src_db.getLatestPath();
- let changed = (this._prev_source_snapshot_path == null ||
- !this._prev_source_snapshot_path.equal(this._source_snapshot_path));
+ this._prevSourceSnapshotPath = this._sourceSnapshotPath;
+ this._sourceSnapshotPath = this._src_db.getLatestPath();
+ let changed = (this._prevSourceSnapshotPath == null ||
+ !this._prevSourceSnapshotPath.equal(this._sourceSnapshotPath));
if (changed)
- print(Format.vprintf("New version is %s", [this._source_snapshot_path.get_path()]))
- if (!this._build_needed)
- this._build_needed = changed;
- if (this._build_needed && !this._build_taskset.isRunning())
- this._run_build();
-
- if (this._full_resolve_needed || this._queued_force_resolve.length > 0) {
- this._fetch();
- }
-
+ print(Format.vprintf("New version is %s", [this._sourceSnapshotPath.get_path()]))
+ if (!this._buildNeeded)
+ this._buildNeeded = changed;
+ this._runBuild();
+ this._runResolve();
this._updateStatus();
},
-
- _run_build: function() {
- let cancellable = null;
- if (this._build_taskset.isRunning()) throw new Error();
- if (!this._build_needed) throw new Error();
- this._build_needed = false;
-
- let snapshotName = this._source_snapshot_path.get_basename();
-
- let workdir = this._build_taskset.prepare();
- let tmpSnapshotPath = workdir.get_child(snapshotName);
- GSystem.file_linkcopy(this._source_snapshot_path, tmpSnapshotPath,
- Gio.FileCopyFlags.OVERWRITE, cancellable);
-
- let version = this._src_db.parseVersionStr(this._source_snapshot_path.get_basename());
- let meta = {'version': version,
- 'version-path': this._snapshot_dir.get_relative_path(this._source_snapshot_path)};
- let metaPath = workdir.get_child('meta.json');
- JsonUtil.writeJsonFileAtomic(metaPath, meta, cancellable);
-
- let args = ['ostbuild', 'build', '--snapshot=' + snapshotName];
-
- let context = new GSystem.SubprocessContext({ argv: args });
- let task = this._build_taskset.start(context,
- cancellable,
- Lang.bind(this, this._onBuildExited));
- print(Format.vprintf("Build task %s started", [task.versionstr]));
-
- this._updateStatus();
+ _onBuildExited: function(buildTaskset, success, msg) {
+ print(Format.vprintf("build exited; success=%s msg=%s", [success, msg]))
+ if (this._buildNeeded)
+ this._runBuild()
+
+ this._updateStatus();
},
- _run_builddisks: function() {
+
+ _runBuild: function() {
let cancellable = null;
-
- if (!this._do_builddisks || this._builddisks_taskset.isRunning())
+ if (this._taskmaster.isTaskQueued(this._buildTaskName))
return;
-
- let args = ['ostbuild', 'build-disks'];
-
- let context = new GSystem.SubprocessContext({ argv: args });
- let task = this._builddisks_taskset.start(context,
- cancellable,
- Lang.bind(this, this._onBuildDisksExited));
- print(Format.vprintf("Builddisks task %s started", [task.versionstr]));
-
- this._updateStatus();
- },
-
- _run_smoke: function() {
- let cancellable = null;
-
- if (!this._do_smoke || this._smoke_taskset.isRunning())
+ if (!this._buildNeeded)
return;
- let args = ['ostbuild', 'qa-smoketest'];
+ this._buildNeeded = false;
- let context = new GSystem.SubprocessContext({ argv: args });
- let task = this._smoke_taskset.start(context,
- cancellable,
- Lang.bind(this, this._onSmokeExited));
- print(Format.vprintf("Smoke task %s started", [task.versionstr]));
+ let snapshotName = this._sourceSnapshotPath.get_basename();
- this._updateStatus();
- },
+ this._taskmaster.pushTask(this._buildTaskName);
- _onBuildExited: function(buildTaskset, success, msg) {
- print(Format.vprintf("build exited; success=%s msg=%s", [success, msg]))
- if (this._build_needed)
- this._run_build()
- if (success)
- this._run_builddisks();
-
- this._updateStatus();
- },
-
- _onBuildDisksExited: function(buildTaskset, success, msg) {
- print(Format.vprintf("builddisks exited; success=%s msg=%s", [success, msg]))
this._updateStatus();
-
- if (success)
- this._run_smoke();
-
- this._updateStatus();
- },
-
- _getBuildDiffForTask: function(task) {
- let cancellable = null;
- if (task.build_diff != undefined)
- return task.build_diff;
- let metaPath = task.path.get_child('meta.json');
- if (!metaPath.query_exists(null)) {
- task.build_diff = null;
- return task.build_diff;
- }
- let meta = JsonUtil.loadJson(metaPath, cancellable);
- let snapshotPath = this._snapshot_dir.get_child(meta['version-path']);
- let prevSnapshotPath = this._src_db.getPreviousPath(snapshotPath);
- if (prevSnapshotPath == null) {
- task.build_diff = null;
- } else {
- task.build_diff = Snapshot.snapshotDiff(this._src_db.loadFromPath(snapshotPath, cancellable),
- this._src_db.loadFromPath(prevSnapshotPath, cancellable));
- }
- return task.build_diff;
- },
-
- _buildHistoryToJson: function() {
- let cancellable = null;
- let history = this._build_taskset.getHistory();
- let l = history.length;
- let MAXITEMS = 5;
- let entries = [];
- for (let i = Math.max(l - MAXITEMS, 0); i >= 0 && i < l; i++) {
- let item = history[i];
- let data = {v: item.versionstr,
- state: item.state,
- timestamp: item.timestamp};
- entries.push(data);
- let metaPath = item.path.get_child('meta.json');
- if (metaPath.query_exists(cancellable)) {
- data['meta'] = JsonUtil.loadJson(metaPath, cancellable);
- }
- data['diff'] = this._getBuildDiffForTask(item);
- }
- return entries;
- },
-
- _writeStatusFile: function() {
- let cancellable = null;
- let status = {'prefix': this.prefix};
- if (this._source_snapshot_path != null) {
- let version = this._src_db.parseVersionStr(this._source_snapshot_path.get_basename());
- status['version'] = version;
- status['version-path'] = this._snapshot_dir.get_relative_path(this._source_snapshot_path);
- } else {
- status['version'] = '';
- }
-
- status['build'] = this._buildHistoryToJson();
-
- if (this._build_proc != null) {
- let buildHistory = this._build_taskset.getHistory();
- let activeBuild = buildHistory[buildHistory.length-1];
- let buildStatus = status['build'];
- let activeBuildJson = buildStatus[buildStatus.length-1];
- let statusPath = activeBuild.path.get_child('status.json');
- if (statusPath.query_exists(null)) {
- activeBuildJson['build-status'] = JsonUtil.loadJson(statusPath);
- }
- }
-
- JsonUtil.writeJsonFileAtomic(this._status_path, status, cancellable);
}
});
diff --git a/src/ostbuild/js/builtins/git_mirror.js b/src/ostbuild/js/builtins/git_mirror.js
index b1101c3..d1c05cb 100644
--- a/src/ostbuild/js/builtins/git_mirror.js
+++ b/src/ostbuild/js/builtins/git_mirror.js
@@ -41,6 +41,7 @@ const GitMirror = new Lang.Class({
this.parser.addArgument('--prefix');
this.parser.addArgument('--manifest');
this.parser.addArgument('--snapshot');
+ this.parser.addArgument('--timeout-sec', { help: "Cache fetch results for provided number of seconds" });
this.parser.addArgument('--fetch', {action:'storeTrue',
help:"Also do a git fetch for components"});
this.parser.addArgument(['-k', '--keep-going'], {action:'storeTrue',
@@ -51,17 +52,13 @@ const GitMirror = new Lang.Class({
execute: function(args, loop, cancellable) {
let parser = new ArgParse.ArgumentParser();
+ if (!args.timeout_sec)
+ args.timeout_sec = 0;
+
if (args.manifest != null) {
- let snapshotData = JsonUtil.loadJson(Gio.File.new_for_path(args.manifest), cancellable);
- let resolvedComponents = [];
- let components = snapshotData['components'];
- for (let i = 0; i < components.length; i++) {
- resolvedComponents.push(BuildUtil.resolveComponent(snapshotData, components[i]));
- }
- snapshotData['components'] = resolvedComponents;
- snapshotData['patches'] = BuildUtil.resolveComponent(snapshotData, snapshotData['patches']);
- snapshotData['base'] = BuildUtil.resolveComponent(snapshotData, snapshotData['base']);
- this._snapshot = new Snapshot.Snapshot(snapshotData, null);
+ let manifestPath = Gio.File.new_for_path(args.manifest)
+ let manifestData = JsonUtil.loadJson(manifestPath, cancellable);
+ this._snapshot = new Snapshot.Snapshot(manifestData, manifestPath, { prepareResolve: true });
} else {
this._initSnapshot(args.prefix, args.snapshot, cancellable);
}
@@ -74,18 +71,15 @@ const GitMirror = new Lang.Class({
}
componentNames.forEach(Lang.bind(this, function (name) {
- let component = this._snapshot.getComponent(name);
- let src = component['src']
- let [keytype, uri] = Vcs.parseSrcKey(src);
- let branch = component['branch'];
- let tag = component['tag'];
- let branchOrTag = branch || tag;
+ let [keytype, uri, branchOrTag] = this._snapshot.getVcsInfo(name);
if (!args.fetch) {
Vcs.ensureVcsMirror(this.mirrordir, keytype, uri, branchOrTag, cancellable);
} else {
print("Running git fetch for " + name);
- Vcs.fetch(this.mirrordir, keytype, uri, branchOrTag, cancellable, {keepGoing:args.keep_going});
+ Vcs.fetch(this.mirrordir, keytype, uri, branchOrTag, cancellable,
+ { keepGoing:args.keep_going,
+ timeoutSec: args.timeout_sec });
}
}));
}
diff --git a/src/ostbuild/js/builtins/make.js b/src/ostbuild/js/builtins/make.js
new file mode 100644
index 0000000..153cdfd
--- /dev/null
+++ b/src/ostbuild/js/builtins/make.js
@@ -0,0 +1,108 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const Task = imports.task;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const Make = new Lang.Class({
+ Name: 'Make',
+ Extends: Builtin.Builtin,
+
+ DESCRIPTION: "Execute tasks",
+
+ _init: function() {
+ this.parent();
+ this.parser.addArgument('taskname');
+ this.parser.addArgument('parameters', { nargs: '*' });
+ },
+
+ execute: function(args, loop, cancellable) {
+ this._loop = loop;
+ this._err = null;
+ let taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
+ { onEmpty: Lang.bind(this, this._onTasksComplete) });
+ this._taskmaster = taskmaster;
+ taskmaster.connect('task-completed', Lang.bind(this, this._onTaskCompleted));
+ let params = {};
+ for (let i = 0; i < args.parameters.length; i++) {
+ let param = args.parameters[i];
+ let idx = param.indexOf('=');
+ if (idx == -1)
+ throw new Error("Invalid key=value syntax");
+ let k = param.substr(0, idx);
+ let v = JSON.parse(param.substr(idx+1));
+ params[k] = v;
+ }
+ taskmaster.pushTask(args.taskname, params);
+ this._console = GSystem.Console.get();
+ if (this._console) {
+ this._console.begin_status_line("", cancellable);
+ GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT, 1, Lang.bind(this, this._idleUpdateStatus));
+ }
+ loop.run();
+ if (this._console)
+ this._console.end_status_line(cancellable);
+ if (this._err)
+ throw new Error("Error: " + this._err);
+ else
+ print("Success!")
+ },
+
+ _idleUpdateStatus: function() {
+ let [success, loadavg, len] = GLib.file_get_contents('/proc/loadavg');
+ loadavg = loadavg.toString();
+ let elts = loadavg.split(' ');
+ let loadAvg = elts[0];
+ let schedulables = elts[3];
+
+ let taskstateList = this._taskmaster.getTaskState();
+ let taskNames = "";
+ for (let i = 0; i < taskstateList.length; i++) {
+ let taskstate = taskstateList[i];
+ taskNames += (taskstate.task.name + " ");
+ }
+
+ this._console.begin_status_line("running: " + taskNames + "; load=" + loadAvg + " sched=" + schedulables, null);
+
+ return true;
+ },
+
+ _onTaskCompleted: function(taskmaster, task, result, error) {
+ print("Task " + task.TaskName + " complete: " + task.dir.get_path());
+ },
+
+ _onTasksComplete: function(success, err) {
+ if (!success)
+ this._err = err;
+ this._loop.quit();
+ }
+});
diff --git a/src/ostbuild/js/builtins/run_task.js b/src/ostbuild/js/builtins/run_task.js
new file mode 100644
index 0000000..4a34c32
--- /dev/null
+++ b/src/ostbuild/js/builtins/run_task.js
@@ -0,0 +1,55 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const Task = imports.task;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const RunTask = new Lang.Class({
+ Name: 'RunTask',
+ Extends: Builtin.Builtin,
+
+ DESCRIPTION: "Internal helper to execute a task",
+
+ _init: function() {
+ this.parent();
+ this.parser.addArgument('taskName');
+ this.parser.addArgument('parameters');
+ },
+
+ execute: function(args, loop, cancellable) {
+ let taskset = Task.TaskSet.prototype.getInstance();
+ let [taskDef, vars] = taskset.getTask(args.taskName);
+ let params = JSON.parse(args.parameters);
+ let instance = new taskDef(null, args.taskName, vars, params);
+ instance.execute(cancellable);
+ }
+});
diff --git a/src/ostbuild/js/jsonutil.js b/src/ostbuild/js/jsonutil.js
index 04a82ca..24deb88 100644
--- a/src/ostbuild/js/jsonutil.js
+++ b/src/ostbuild/js/jsonutil.js
@@ -27,6 +27,44 @@ function writeJsonToStream(stream, data, cancellable) {
stream.write_bytes(new GLib.Bytes(buf), cancellable);
}
+function writeJsonToStreamAsync(stream, data, cancellable, onComplete) {
+ let buf = JSON.stringify(data, null, " ");
+ stream.write_bytes_async(new GLib.Bytes(buf), GLib.PRIORITY_DEFAULT,
+ cancellable, function(stream, result) {
+ let err = null;
+ try {
+ stream.write_bytes_finish(result);
+ } catch (e) {
+ err = e;
+ }
+ onComplete(err != null, err);
+ });
+}
+
+function loadJsonFromStream(stream, cancellable) {
+ let membuf = Gio.MemoryOutputStream.new_resizable();
+ membuf.splice(stream, Gio.OutputStreamSpliceFlags.CLOSE_TARGET, cancellable);
+ let bytes = membuf.steal_as_bytes();
+ return JSON.parse(bytes.toArray().toString());
+}
+
+function loadJsonFromStreamAsync(stream, cancellable, onComplete) {
+ let membuf = Gio.MemoryOutputStream.new_resizable();
+ membuf.splice_async(stream, Gio.OutputStreamSpliceFlags.CLOSE_TARGET, GLib.PRIORITY_DEFAULT,
+ cancellable, function(stream, result) {
+ let err = null;
+ let res = null;
+ try {
+ stream.splice_finish(result);
+ let bytes = membuf.steal_as_bytes();
+ res = JSON.parse(bytes.toArray().toString());
+ } catch (e) {
+ err = e;
+ }
+ onComplete(res, err);
+ });
+}
+
function writeJsonFileAtomic(path, data, cancellable) {
let s = path.replace(null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable);
writeJsonToStream(s, data, cancellable);
diff --git a/src/ostbuild/js/main.js b/src/ostbuild/js/main.js
index ae92800..0f1e2e2 100755
--- a/src/ostbuild/js/main.js
+++ b/src/ostbuild/js/main.js
@@ -26,7 +26,9 @@ const BUILTINS = ['autobuilder',
'resolve',
'build',
'build-disks',
+ 'make',
'shell',
+ 'run-task',
'qa-make-disk',
'qa-pull-deploy',
'qa-smoketest'];
diff --git a/src/ostbuild/js/procutil.js b/src/ostbuild/js/procutil.js
index 0af1ab3..3c30ab3 100644
--- a/src/ostbuild/js/procutil.js
+++ b/src/ostbuild/js/procutil.js
@@ -92,7 +92,7 @@ function _runSyncGetOutputInternal(argv, cancellable, params, splitLines) {
let [line, len] = dataIn.read_line_utf8(cancellable);
if (line == null)
break;
- result += line;
+ result += (line + '\n');
}
}
_wait_sync_check_internal(proc, cancellable);
diff --git a/src/ostbuild/js/snapshot.js b/src/ostbuild/js/snapshot.js
index 51dca5f..fbe760c 100644
--- a/src/ostbuild/js/snapshot.js
+++ b/src/ostbuild/js/snapshot.js
@@ -21,14 +21,17 @@ const Lang = imports.lang;
const JsonDB = imports.jsondb;
const JsonUtil = imports.jsonutil;
+const Vcs = imports.vcs;
const Params = imports.params;
function _componentDict(snapshot) {
let r = {};
let components = snapshot['components'];
- for (let i = 0; i< components.length; i++) {
+ for (let i = 0; i < components.length; i++) {
let component = components[i];
let name = component['name'];
+ if (r[name])
+ throw new Error("Duplicate component name " + name);
r[name] = component;
}
let patches = snapshot['patches'];
@@ -66,15 +69,69 @@ function snapshotDiff(a, b) {
const Snapshot = new Lang.Class({
Name: 'Snapshot',
- _init: function(data, path) {
+ _init: function(data, path, params) {
+ params = Params.parse(params, { prepareResolve: false });
this.data = data;
this.path = path;
+ if (params.prepareResolve) {
+ data['patches'] = this._resolveComponent(data, data['patches']);
+ data['base'] = this._resolveComponent(data, data['base']);
+ for (let i = 0; i < data['components'].length; i++) {
+ let component = this._resolveComponent(data, data['components'][i]);
+ data['components'][i] = component;
+ }
+ }
this._componentDict = _componentDict(data);
this._componentNames = [];
for (let k in this._componentDict)
this._componentNames.push(k);
},
+ _resolveComponent: function(manifest, componentMeta) {
+ let result = {};
+ Lang.copyProperties(componentMeta, result);
+ let origSrc = componentMeta['src'];
+
+ let didExpand = false;
+ let vcsConfig = manifest['vcsconfig'];
+ for (let vcsprefix in vcsConfig) {
+ let expansion = vcsConfig[vcsprefix];
+ let prefix = vcsprefix + ':';
+ if (origSrc.indexOf(prefix) == 0) {
+ result['src'] = expansion + origSrc.substr(prefix.length);
+ didExpand = true;
+ break;
+ }
+ }
+
+ let name = componentMeta['name'];
+ let src, idx, name;
+ if (name == undefined) {
+ if (didExpand) {
+ src = origSrc;
+ idx = src.lastIndexOf(':');
+ name = src.substr(idx+1);
+ } else {
+ src = result['src'];
+ idx = src.lastIndexOf('/');
+ name = src.substr(idx+1);
+ }
+ let i = name.lastIndexOf('.git');
+ if (i != -1 && i == name.length - 4) {
+ name = name.substr(0, name.length - 4);
+ }
+ name = name.replace(/\//g, '-');
+ result['name'] = name;
+ }
+
+ let branchOrTag = result['branch'] || result['tag'];
+ if (!branchOrTag) {
+ result['branch'] = 'master';
+ }
+
+ return result;
+ },
+
_expandComponent: function(component) {
let r = {};
Lang.copyProperties(component, r);
@@ -98,6 +155,10 @@ const Snapshot = new Lang.Class({
return this._componentNames;
},
+ getComponentMap: function() {
+ return this._componentDict;
+ },
+
getComponent: function(name, allowNone) {
let r = this._componentDict[name] || null;
if (!r && !allowNone)
@@ -118,5 +179,15 @@ const Snapshot = new Lang.Class({
getExpanded: function(name) {
return this._expandComponent(this.getComponent(name));
+ },
+
+ getVcsInfo: function(name) {
+ let component = this.getComponent(name);
+ let src = component['src']
+ let [keytype, uri] = Vcs.parseSrcKey(src);
+ let branch = component['branch'];
+ let tag = component['tag'];
+ let branchOrTag = branch || tag;
+ return [keytype, uri, branchOrTag];
}
});
diff --git a/src/ostbuild/js/task.js b/src/ostbuild/js/task.js
new file mode 100644
index 0000000..665033c
--- /dev/null
+++ b/src/ostbuild/js/task.js
@@ -0,0 +1,345 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const format = imports.format;
+const Lang = imports.lang;
+const Signals = imports.signals;
+
+const GSystem = imports.gi.GSystem;
+const Config = imports.config;
+const Params = imports.params;
+const JsonUtil = imports.jsonutil;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const BuildUtil = imports.buildutil;
+
+const VERSION_RE = /(\d+)\.(\d+)/;
+
+var _tasksetInstance = null;
+const TaskSet = new Lang.Class({
+ Name: 'TaskSet',
+
+ _init: function() {
+ this._tasks = [];
+ let taskdir = Gio.File.new_for_path(GLib.getenv('OSTBUILD_DATADIR')).resolve_relative_path('js/tasks');
+ let denum = taskdir.enumerate_children('standard::*', 0, null);
+ let finfo;
+
+ for (let taskmodname in imports.tasks) {
+ let taskMod = imports.tasks[taskmodname];
+ for (let defname in taskMod) {
+ if (defname.indexOf('Task') !== 0
+ || defname == 'Task')
+ continue;
+ let cls = taskMod[defname];
+ this.register(cls);
+ }
+ }
+ },
+
+ register: function(taskdef) {
+ this._tasks.push(taskdef);
+ },
+
+ getAllTasks: function() {
+ return this._tasks;
+ },
+
+ getTask: function(taskName, params) {
+ params = Params.parse(params, { allowNone: false })
+ for (let i = 0; i < this._tasks.length; i++) {
+ let taskDef = this._tasks[i];
+ let pattern = taskDef.prototype.TaskPattern;
+ let re = pattern[0];
+ let match = re.exec(taskName);
+ if (!match)
+ continue;
+ let vars = {};
+ for (let i = 1; i < pattern.length; i++) {
+ vars[pattern[i]] = match[i];
+ }
+ return [taskDef, vars];
+ }
+ if (!params.allowNone)
+ throw new Error("No task definition matches " + taskName);
+ return null;
+ },
+
+ getInstance: function() {
+ if (!_tasksetInstance)
+ _tasksetInstance = new TaskSet();
+ return _tasksetInstance;
+ }
+});
+
+const TaskMaster = new Lang.Class({
+ Name: 'TaskMaster',
+
+ _init: function(path, params) {
+ params = Params.parse(params, {onEmpty: null});
+ this.path = path;
+ this.maxConcurrent = GLib.get_num_processors();
+ this._onEmpty = params.onEmpty;
+ this.cancellable = null;
+ this._idleRecalculateId = 0;
+ this._executing = [];
+ this._pendingTasksList = [];
+ this._seenTasks = {};
+ this._taskErrors = {};
+ this._caughtError = false;
+
+ this._taskset = TaskSet.prototype.getInstance();
+ },
+
+ pushTask: function(taskName, parameters) {
+ if (this.isTaskQueued(taskName))
+ return;
+ let [taskDef, vars] = this._taskset.getTask(taskName);
+ let instance = new taskDef(this, taskName, vars, parameters);
+ instance.onComplete = Lang.bind(this, this._onComplete, instance);
+ this._pendingTasksList.push(instance);
+ this._queueRecalculate();
+ },
+
+ isTaskQueued: function(taskName) {
+ for (let i = 0; i < this._pendingTasksList.length; i++) {
+ let pending = this._pendingTasksList[i];
+ if (pending.TaskName == taskName)
+ return true;
+ }
+ for (let i = 0; i < this._executing.length; i++) {
+ let executingTask = this._executing[i];
+ if (executingTask.TaskName == taskName)
+ return true;
+ }
+ return false;
+ },
+
+ getTaskState: function() {
+ let r = [];
+ for (let i = 0; i < this._pendingTasksList.length; i++) {
+ r.push({running: false, task: this._pendingTasksList[i] });
+ }
+ for (let i = 0; i < this._executing.length; i++) {
+ r.push({running: true, task: this._executing[i] });
+ }
+ return r;
+ },
+
+ _queueRecalculate: function() {
+ if (this._idleRecalculateId > 0)
+ return;
+ this._idleRecalculateId = GLib.idle_add(GLib.PRIORITY_DEFAULT, Lang.bind(this, this._recalculate));
+ },
+
+ _recalculate: function() {
+ this._idleRecalculateId = 0;
+
+ if (this._executing.length == 0 &&
+ this._pendingTasksList.length == 0) {
+ this._onEmpty(true, null);
+ return;
+ } else if (this._pendingTasksList.length == 0) {
+ return;
+ }
+
+ this._reschedule();
+ },
+
+ _onComplete: function(success, error, task) {
+ this.emit('task-complete', task, success, error);
+ let idx = -1;
+ for (let i = 0; i < this._executing.length; i++) {
+ let executingTask = this._executing[i];
+ if (executingTask !== task)
+ continue;
+ idx = i;
+ break;
+ }
+ if (idx == -1)
+ throw new Error("TaskMaster: Internal error - Failed to find completed task:" + task.TaskName);
+ this._executing.splice(idx, 1);
+ this._queueRecalculate();
+ },
+
+ _reschedule: function() {
+ while (this._executing.length < this.maxConcurrent &&
+ this._pendingTasksList.length > 0) {
+ let task = this._pendingTasksList.shift();
+ task._executeInSubprocessInternal(this.cancellable);
+ this._executing.push(task);
+ }
+ }
+});
+Signals.addSignalMethods(TaskMaster.prototype);
+
+const TaskDef = new Lang.Class({
+ Name: 'TaskDef',
+
+ TaskPattern: null,
+
+ PreserveStdout: true,
+ RetainFailed: 1,
+ RetainSuccess: 5,
+
+ DefaultParameters: {},
+
+ _VERSION_RE: /(\d+\d\d\d\d)\.(\d+)/,
+
+ _init: function(taskmaster, name, vars, parameters) {
+ this.taskmaster = taskmaster;
+ this.name = name;
+ this.vars = vars;
+ this.parameters = Params.parse(parameters, this.DefaultParameters);
+
+ this.config = Config.get();
+ this.workdir = Gio.File.new_for_path(this.config.getGlobal('workdir'));
+ this.resultdir = this.workdir.get_child('results');
+ this.mirrordir = Gio.File.new_for_path(this.config.getGlobal('mirrordir'));
+ this.libdir = Gio.File.new_for_path(GLib.getenv('OSTBUILD_LIBDIR'));
+ this.repo = this.workdir.get_child('repo');
+ },
+
+ getDepends: function() {
+ return [];
+ },
+
+ _getResultDb: function(taskname) {
+ let path = this.resultdir.resolve_relative_path(taskname);
+ return new JsonDB.JsonDB(path);
+ },
+
+ _loadVersionsFrom: function(dir, cancellable) {
+ let e = dir.enumerate_children('standard::*', Gio.FileQueryInfoFlags.NONE, cancellable);
+ let info;
+ let results = [];
+ while ((info = e.next_file(cancellable)) != null) {
+ let name = info.get_name();
+ let match = this._VERSION_RE.exec(name);
+ if (!match)
+ continue;
+ results.push(name);
+ }
+ results.sort(BuildUtil.compareVersions);
+ return results;
+ },
+
+ _cleanOldVersions: function(dir, retain, cancellable) {
+ let versions = this._loadVersionsFrom(dir, cancellable);
+ while (versions.length > retain) {
+ let child = dir.get_child(versions.shift());
+ GSystem.shutil_rm_rf(child, cancellable);
+ }
+ },
+
+ execute: function(cancellable) {
+ throw new Error("Not implemented");
+ },
+
+ _executeInSubprocessInternal: function(cancellable) {
+ this._cancellable = cancellable;
+
+ this.dir = this.taskmaster.path.resolve_relative_path(this.name);
+ GSystem.file_ensure_directory(this.dir, true, cancellable);
+
+ let allVersions = [];
+
+ this._successDir = this.dir.get_child('successful');
+ GSystem.file_ensure_directory(this._successDir, true, cancellable);
+ let successVersions = this._loadVersionsFrom(this._successDir, cancellable);
+ for (let i = 0; i < successVersions.length; i++) {
+ allVersions.push([true, successVersions[i]]);
+ }
+
+ this._failedDir = this.dir.get_child('failed');
+ GSystem.file_ensure_directory(this._failedDir, true, cancellable);
+ let failedVersions = this._loadVersionsFrom(this._failedDir, cancellable);
+ for (let i = 0; i < failedVersions.length; i++) {
+ allVersions.push([false, failedVersions[i]]);
+ }
+
+ allVersions.sort(function (a, b) {
+ let [successA, versionA] = a;
+ let [successB, versionB] = b;
+ return BuildUtil.compareVersions(versionA, versionB);
+ });
+
+ let currentTime = GLib.DateTime.new_now_utc();
+
+ let currentYmd = Format.vprintf('%d%02d%02d', [currentTime.get_year(),
+ currentTime.get_month(),
+ currentTime.get_day_of_month()]);
+ let version = null;
+ if (allVersions.length > 0) {
+ let [lastSuccess, lastVersion] = allVersions[allVersions.length-1];
+ let match = this._VERSION_RE.exec(lastVersion);
+ if (!match) throw new Error();
+ let lastYmd = match[1];
+ let lastSerial = match[2];
+ if (lastYmd == currentYmd) {
+ version = currentYmd + '.' + (parseInt(lastSerial) + 1);
+ }
+ }
+ if (version === null) {
+ version = currentYmd + '.0';
+ }
+
+ this._workdir = this.dir.get_child(version);
+ GSystem.shutil_rm_rf(this._workdir, cancellable);
+ GSystem.file_ensure_directory(this._workdir, true, cancellable);
+
+ let baseArgv = ['ostbuild', 'run-task', this.name, JSON.stringify(this.parameters)];
+ let context = new GSystem.SubprocessContext({ argv: baseArgv });
+ context.set_cwd(this._workdir.get_path());
+ context.set_stdin_disposition(GSystem.SubprocessStreamDisposition.PIPE);
+ if (this.PreserveStdout) {
+ let outPath = this._workdir.get_child('output.txt');
+ context.set_stdout_file_path(outPath.get_path());
+ context.set_stderr_disposition(GSystem.SubprocessStreamDisposition.STDERR_MERGE);
+ } else {
+ context.set_stdout_disposition(GSystem.SubprocessStreamDisposition.NULL);
+ let errPath = this._workdir.get_child('errors.txt');
+ context.set_stderr_file_path(errPath.get_path());
+ }
+ this._proc = new GSystem.Subprocess({ context: context });
+ this._proc.init(cancellable);
+
+ this._proc.wait(cancellable, Lang.bind(this, this._onChildExited));
+ },
+
+ _onChildExited: function(proc, result) {
+ let [success, errmsg] = ProcUtil.asyncWaitCheckFinish(proc, result);
+ let target;
+ if (!success) {
+ target = this._failedDir.get_child(this._workdir.get_basename());
+ GSystem.file_rename(this._workdir, target, null);
+ this.dir = target;
+ this._cleanOldVersions(this._failedDir, this.RetainFailed, null);
+ this.onComplete(success, errmsg);
+ } else {
+ target = this._successDir.get_child(this._workdir.get_basename());
+ GSystem.file_rename(this._workdir, target, null);
+ this.dir = target;
+ this._cleanOldVersions(this._successDir, this.RetainSuccess, null);
+ this.onComplete(success, null);
+ }
+ // Also remove any old interrupted versions
+ this._cleanOldVersions(this.dir, 0, null);
+ }
+});
diff --git a/src/ostbuild/js/tasks/task-bdiff.js b/src/ostbuild/js/tasks/task-bdiff.js
new file mode 100644
index 0000000..0a96ae1
--- /dev/null
+++ b/src/ostbuild/js/tasks/task-bdiff.js
@@ -0,0 +1,153 @@
+// Copyright (C) 2011,2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const Task = imports.task;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const StreamUtil = imports.streamutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const TaskBdiff = new Lang.Class({
+ Name: "TaskBdiff",
+ Extends: Task.TaskDef,
+
+ TaskPattern: [/bdiff\/(.*?)$/, 'prefix'],
+
+ _gitLogToJson: function(repoDir, specification) {
+ let log = ProcUtil.runSyncGetOutputLines(['git', 'log', '--format=email', specification],
+ null,
+ { cwd: repoDir });
+ let r = [];
+ if (log.length == 0)
+ return r;
+ let currentItem = null;
+ let parsingHeaders = false;
+ let fromRegex = /^From ([0-9a-f]{40}) /;
+ for (let i = 0; i < log.length; i++) {
+ let line = log[i];
+ let match = fromRegex.exec(line);
+ if (match) {
+ if (currentItem !== null) {
+ r.push(currentItem);
+ }
+ currentItem = {'Checksum': match[1]};
+ parsingHeaders = true;
+ } else if (parsingHeaders) {
+ if (line.length == 0) {
+ parsingHeaders = false;
+ } else {
+ let idx = line.indexOf(':');
+ let k = line.substr(0, idx);
+ let v = line.substr(idx+1);
+ currentItem[k] = v;
+ }
+ }
+ }
+ return r;
+ },
+
+ _diffstat: function(repoDir, specification) {
+ return ProcUtil.runSyncGetOutputUTF8(['git', 'diff', '--stat', specification], null,
+ { cwd: repoDir });
+ },
+
+ execute: function(cancellable) {
+ let prefix = this.vars['prefix'];
+
+ this.subworkdir = Gio.File.new_for_path('.');
+
+ let builddb = this._getResultDb('build/' + prefix);
+ let latestPath = builddb.getLatestPath();
+ if (!latestPath)
+ throw new Error("No builds!")
+ let latestBuildVersion = builddb.parseVersionStr(latestPath.get_basename());
+
+ let previousPath = builddb.getPreviousPath(latestPath);
+ if (!previousPath)
+ throw new Error("No build previous to " + latestBuildVersion)
+
+ let latestBuildData = builddb.loadFromPath(latestPath, cancellable);
+ let latestBuildSnapshot = new Snapshot.Snapshot(latestBuildData['snapshot'], null);
+ let previousBuildData = builddb.loadFromPath(previousPath, cancellable);
+ let previousBuildSnapshot = new Snapshot.Snapshot(previousBuildData['snapshot'], null);
+
+ let added = [];
+ let modified = [];
+ let removed = [];
+
+ let result = {fromBuildVersion: builddb.parseVersionStr(previousPath.get_basename()),
+ toBuildVersion: builddb.parseVersionStr(latestPath.get_basename()),
+ fromSrcVersion: builddb.parseVersionStr(previousBuildData['snapshotName']),
+ toSrcVersion: builddb.parseVersionStr(latestBuildData['snapshotName']),
+ added: added,
+ modified: modified,
+ removed: removed};
+
+ let modifiedNames = [];
+
+ let latestComponentMap = latestBuildSnapshot.getComponentMap();
+ let previousComponentMap = previousBuildSnapshot.getComponentMap();
+ for (let componentName in latestComponentMap) {
+ let componentA = latestBuildSnapshot.getComponent(componentName);
+ let componentB = previousBuildSnapshot.getComponent(componentName, true);
+
+ if (componentB === null)
+ added.push(componentName);
+ else if (componentB.revision != componentA.revision)
+ modifiedNames.push(componentName);
+ }
+ for (let componentName in previousComponentMap) {
+ let componentA = latestBuildSnapshot.getComponent(componentName, true);
+
+ if (componentA === null)
+ removed.push(componentName);
+ }
+
+ for (let i = 0; i < modifiedNames.length; i++) {
+ let componentName = modifiedNames[i];
+ let latestComponent = latestBuildSnapshot.getComponent(componentName);
+ let previousComponent = previousBuildSnapshot.getComponent(componentName);
+ let latestRevision = latestComponent.revision;
+ let previousRevision = previousComponent.revision;
+ let [keytype, uri, branchOrTag] = latestBuildSnapshot.getVcsInfo(componentName);
+ let mirrordir = Vcs.getMirrordir(this.mirrordir, keytype, uri);
+
+ let gitlog = this._gitLogToJson(mirrordir, previousRevision + '...' + latestRevision);
+ let diffstat = this._diffstat(mirrordir, previousRevision + '..' + latestRevision);
+ modified.push({ previous: previousComponent,
+ latest: latestComponent,
+ gitlog: gitlog,
+ diffstat: diffstat });
+ }
+
+ let bdiffdb = this._getResultDb('bdiff/' + prefix);
+ bdiffdb.store(result, cancellable);
+ }
+});
diff --git a/src/ostbuild/js/builtins/build.js b/src/ostbuild/js/tasks/task-build.js
similarity index 92%
rename from src/ostbuild/js/builtins/build.js
rename to src/ostbuild/js/tasks/task-build.js
index 7c54a8a..c68a097 100644
--- a/src/ostbuild/js/builtins/build.js
+++ b/src/ostbuild/js/tasks/task-build.js
@@ -23,7 +23,7 @@ const Format = imports.format;
const GSystem = imports.gi.GSystem;
const Builtin = imports.builtin;
-const SubTask = imports.subtask;
+const Task = imports.task;
const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const StreamUtil = imports.streamutil;
@@ -35,13 +35,13 @@ const Vcs = imports.vcs;
const ArgParse = imports.argparse;
const OPT_COMMON_CFLAGS = {'i686': '-O2 -g -m32 -march=i686 -mtune=atom -fasynchronous-unwind-tables',
- 'x86_64': '-O2 -g -m64 -mtune=generic'}
+ 'x86_64': '-O2 -g -m64 -mtune=generic'};
-const Build = new Lang.Class({
- Name: "Build",
- Extends: Builtin.Builtin,
+const TaskBuild = new Lang.Class({
+ Name: "TaskBuild",
+ Extends: Task.TaskDef,
- DESCRIPTION: "Build multiple components and generate trees",
+ TaskPattern: [/build\/(.*?)$/, 'prefix'],
_resolveRefs: function(refs) {
if (refs.length == 0)
@@ -290,6 +290,7 @@ const Build = new Lang.Class({
let prefix = this._snapshot.data['prefix'];
let buildname = Format.vprintf('%s/%s/%s', [prefix, basename, architecture]);
+ let unixBuildname = buildname.replace(/\//g, '_');
let buildRef = 'components/' + buildname;
let currentVcsVersion = component['revision'];
@@ -320,16 +321,13 @@ const Build = new Lang.Class({
let patchdir;
if (expandedComponent['patches']) {
let patchesRevision = expandedComponent['patches']['revision'];
- if (this.args.patches_path) {
- patchdir = Gio.File.new_for_path(this.args.patches_path);
- } else if (this._cachedPatchdirRevision == patchesRevision) {
+ if (this._cachedPatchdirRevision == patchesRevision) {
patchdir = this.patchdir;
} else {
patchdir = Vcs.checkoutPatches(this.mirrordir,
this.patchdir,
expandedComponent,
- cancellable,
- {patchesPath: this.args.patches_path});
+ cancellable);
this._cachedPatchdirRevision = patchesRevision;
}
if ((previousMetadata != null) &&
@@ -366,33 +364,31 @@ const Build = new Lang.Class({
}
}
- let taskdir = this.workdir.get_child('tasks');
- let buildTaskset = new SubTask.TaskSet(taskdir.get_child(buildname));
+ let cwd = Gio.File.new_for_path('.');
+ let buildWorkdir = cwd.get_child('tmp-' + unixBuildname);
+ GSystem.file_ensure_directory(buildWorkdir, true, cancellable);
- let workdir = buildTaskset.prepare();
-
- let tempMetadataPath = workdir.get_child('_ostbuild-meta.json');
+ let tempMetadataPath = buildWorkdir.get_child('_ostbuild-meta.json');
JsonUtil.writeJsonFileAtomic(tempMetadataPath, expandedComponent, cancellable);
- let componentSrc = workdir.get_child(basename);
+ let componentSrc = buildWorkdir.get_child(basename);
let childArgs = ['ostbuild', 'checkout', '--snapshot=' + this._snapshot.path.get_path(),
'--checkoutdir=' + componentSrc.get_path(),
'--metadata-path=' + tempMetadataPath.get_path(),
'--overwrite', basename];
- if (this.args.patches_path)
- childArgs.push('--patches-path=' + this.args.patches_path);
- else if (patchdir)
+ if (patchdir) {
childArgs.push('--patches-path=' + patchdir.get_path());
- ProcUtil.runSync(childArgs, cancellable, { logInitiation: true });
+ }
+ ProcUtil.runSync(childArgs, cancellable);
GSystem.file_unlink(tempMetadataPath, cancellable);
- let componentResultdir = workdir.get_child('results');
+ let componentResultdir = buildWorkdir.get_child('results');
GSystem.file_ensure_directory(componentResultdir, true, cancellable);
- let rootdir = this._composeBuildroot(workdir, basename, architecture, cancellable);
+ let rootdir = this._composeBuildroot(buildWorkdir, basename, architecture, cancellable);
- let tmpdir=workdir.get_child('tmp');
+ let tmpdir=buildWorkdir.get_child('tmp');
GSystem.file_ensure_directory(tmpdir, true, cancellable);
let srcCompileOnePath = this.libdir.get_child('ostree-build-compile-one');
@@ -424,26 +420,11 @@ const Build = new Lang.Class({
let context = new GSystem.SubprocessContext({ argv: childArgs });
context.set_environment(ProcUtil.objectToEnvironment(envCopy));
-
- let mainContext = new GLib.MainContext();
- mainContext.push_thread_default();
- let loop = GLib.MainLoop.new(mainContext, true);
- let t;
- try {
- t = buildTaskset.start(context, cancellable, Lang.bind(this, this._onBuildComplete, loop));
- print("Started child process " + context.argv.map(GLib.shell_quote).join(' '));
- loop.run();
- } finally {
- mainContext.pop_thread_default();
- }
- let buildSuccess = this._currentBuildSucceded;
- let msg = this._currentBuildSuccessMsg;
-
- if (!buildSuccess) {
- this._analyzeBuildFailure(t, architecture, component, componentSrc,
- currentVcsVersion, previousVcsVersion, cancellable);
- throw new Error("Build failure in component " + buildname + " : " + msg);
- }
+
+ let proc = new GSystem.Subprocess({ context: context });
+ proc.init(cancellable);
+ print("Started child process " + context.argv.map(GLib.shell_quote).join(' '));
+ proc.wait_sync_check(cancellable);
let recordedMetaPath = componentResultdir.get_child('_ostbuild-meta.json');
JsonUtil.writeJsonFileAtomic(recordedMetaPath, expandedComponent, cancellable);
@@ -472,7 +453,7 @@ const Build = new Lang.Class({
if (statoverridePath != null)
GSystem.file_unlink(statoverridePath, cancellable);
- GSystem.shutil_rm_rf(tmpdir, cancellable);
+ GSystem.shutil_rm_rf(buildWorkdir, cancellable);
let ostreeRevision = this._saveComponentBuild(buildname, expandedComponent, cancellable);
@@ -485,8 +466,7 @@ const Build = new Lang.Class({
let runtimeName = 'bases/' + base['runtime'];
let develName = 'bases/' + base['devel'];
- let rootdir = this.workdir.get_child('roots');
- let composeRootdir = rootdir.get_child(target['name']);
+ let composeRootdir = this.subworkdir.get_child(target['name']);
GSystem.shutil_rm_rf(composeRootdir, cancellable);
GSystem.file_ensure_directory(composeRootdir, true, cancellable);
@@ -623,8 +603,7 @@ const Build = new Lang.Class({
Lang.copyProperties(BuildUtil.BUILD_ENV, env);
env['DL_DIR'] = downloads.get_path();
env['SSTATE_DIR'] = sstateDir.get_path();
- ProcUtil.runSync(cmd, cancellable, { logInitiation: true,
- env:ProcUtil.objectToEnvironment(env) });
+ ProcUtil.runSync(cmd, cancellable, {env:ProcUtil.objectToEnvironment(env)});
let componentTypes = ['runtime', 'devel'];
for (let i = 0; i < componentTypes.length; i++) {
@@ -642,26 +621,30 @@ const Build = new Lang.Class({
builtRevisionPath.replace_contents(basemeta['revision'], null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable);
},
- _init: function() {
- this.parent();
- this.parser.addArgument('--prefix');
- this.parser.addArgument('--snapshot');
- this.parser.addArgument('--patches-path');
+ execute: function(cancellable) {
+ let prefix = this.vars['prefix'];
+
+ this.subworkdir = Gio.File.new_for_path('.');
this.forceBuildComponents = {};
this.cachedPatchdirRevision = null;
- },
-
- execute: function(args, loop, cancellable) {
- this._initSnapshot(args.prefix, args.snapshot, cancellable);
- this.args = args;
+
+ this.prefix = prefix;
+ this.patchdir = this.workdir.get_child('patches');
+ let snapshotDir = this.workdir.get_child('snapshots');
+ let srcdb = new JsonDB.JsonDB(snapshotDir.get_child(prefix));
+ let snapshotPath = srcdb.getLatestPath();
+ let workingSnapshotPath = this.subworkdir.get_child(snapshotPath.get_basename());
+ GSystem.file_linkcopy(snapshotPath, workingSnapshotPath, Gio.FileCopyFlags.OVERWRITE,
+ cancellable);
+ let data = srcdb.loadFromPath(workingSnapshotPath, cancellable);
+ this._snapshot = new Snapshot.Snapshot(data, workingSnapshotPath);
let components = this._snapshot.data['components'];
- let buildresultDir = this.workdir.get_child('builds').get_child(this.prefix);
- let builddb = new JsonDB.JsonDB(buildresultDir);
+ let builddb = this._getResultDb('build/' + this.prefix);
- let targetSourceVersion = builddb.parseVersionStr(this._snapshot.path.get_basename())
+ let targetSourceVersion = builddb.parseVersionStr(this._snapshot.path.get_basename());
let haveLocalComponent = false;
for (let i = 0; i < components.length; i++) {
diff --git a/src/ostbuild/js/builtins/build_disks.js b/src/ostbuild/js/tasks/task-builddisks.js
similarity index 55%
rename from src/ostbuild/js/builtins/build_disks.js
rename to src/ostbuild/js/tasks/task-builddisks.js
index dccafeb..70170cf 100644
--- a/src/ostbuild/js/builtins/build_disks.js
+++ b/src/ostbuild/js/tasks/task-builddisks.js
@@ -25,6 +25,7 @@ const GSystem = imports.gi.GSystem;
const Builtin = imports.builtin;
const ArgParse = imports.argparse;
+const Task = imports.task;
const ProcUtil = imports.procutil;
const LibQA = imports.libqa;
const JsonDB = imports.jsondb;
@@ -32,29 +33,38 @@ const Config = imports.config;
const JsonUtil = imports.jsonutil;
const GuestFish = imports.guestfish;
-const loop = GLib.MainLoop.new(null, true);
+const TaskBuildDisks = new Lang.Class({
+ Name: 'TaskBuildDisks',
+ Extends: Task.TaskDef,
-const BuildDisks = new Lang.Class({
- Name: 'BuildDisks',
- Extends: Builtin.Builtin,
+ TaskPattern: [/builddisks\/(.*?)$/, 'prefix'],
- DESCRIPTION: "Generate disk images",
+ execute: function(cancellable) {
+ this.prefix = this.vars['prefix'];
- execute: function(args, loop, cancellable) {
- this._initPrefix(null);
+ this.subworkdir = Gio.File.new_for_path('.');
this.imageDir = this.workdir.get_child('images').get_child(this.prefix);
this.currentImageLink = this.imageDir.get_child('current');
this.previousImageLink = this.imageDir.get_child('previous');
GSystem.file_ensure_directory(this.imageDir, true, cancellable);
- let buildresultDir = this.workdir.get_child('builds').get_child(this.prefix);
- let builddb = new JsonDB.JsonDB(buildresultDir);
+ let builddb = this._getResultDb('build/' + this.prefix);
let latestPath = builddb.getLatestPath();
let buildVersion = builddb.parseVersionStr(latestPath.get_basename());
this._buildData = builddb.loadFromPath(latestPath, cancellable);
+ let targetImageDir = this.imageDir.get_child(buildVersion);
+
+ if (targetImageDir.query_exists(null)) {
+ print("Already created " + targetImageDir.get_path());
+ return;
+ }
+
+ let newImageDir = this.subworkdir.get_child('images');
+ GSystem.file_ensure_directory(newImageDir, true, cancellable);
+
let targets = this._buildData['targets'];
// Special case the default target - we do a pull, then clone
@@ -63,43 +73,54 @@ const BuildDisks = new Lang.Class({
// copying data via libguestfs repeatedly.
let defaultTarget = this._buildData['snapshot']['default-target'];
let defaultRevision = this._buildData['targets'][defaultTarget];
- this._defaultDiskPath = this._diskPathForTarget(defaultTarget, false);
+ let defaultTargetDiskName = this._diskNameForTarget(defaultTarget);
+
+ let currentDefaultDiskPath = this.currentImageLink.get_child(defaultTargetDiskName);
- let tmppath = this._defaultDiskPath.get_parent().get_child(this._defaultDiskPath.get_basename() + '.tmp');
- GSystem.shutil_rm_rf(tmppath, cancellable);
+ let tmpDefaultDiskPath = newImageDir.get_child(defaultTargetDiskName);
+ GSystem.shutil_rm_rf(tmpDefaultDiskPath, cancellable);
- if (!this._defaultDiskPath.query_exists(null)) {
- LibQA.createDisk(tmppath, cancellable);
+ if (!currentDefaultDiskPath.query_exists(null)) {
+ LibQA.createDisk(tmpDefaultDiskPath, cancellable);
} else {
- LibQA.copyDisk(this._defaultDiskPath, tmppath, cancellable);
+ LibQA.copyDisk(currentDefaultDiskPath, tmpDefaultDiskPath, cancellable);
}
let osname = this._buildData['snapshot']['osname'];
- ProcUtil.runSync(['ostbuild', 'qa-pull-deploy', tmppath.get_path(),
+ ProcUtil.runSync(['ostbuild', 'qa-pull-deploy', tmpDefaultDiskPath.get_path(),
this.repo.get_path(), osname, defaultTarget, defaultRevision],
cancellable, { logInitiation: true });
- GSystem.file_rename(tmppath, this._defaultDiskPath, cancellable);
-
for (let targetName in targets) {
if (targetName == defaultTarget)
continue;
let targetRevision = this._buildData['targets'][targetName];
- let diskPath = this._diskPathForTarget(targetName, true);
- tmppath = diskPath.get_parent().get_child(diskPath.get_basename() + '.tmp');
+ let diskName = this._diskNameForTarget(targetName, true);
+ let tmppath = newImageDir.get_child(diskName);
GSystem.shutil_rm_rf(tmppath, cancellable);
- LibQA.createDiskSnapshot(this._defaultDiskPath, tmppath, cancellable);
+ LibQA.createDiskSnapshot(tmpDefaultDiskPath, tmppath, cancellable);
ProcUtil.runSync(['ostbuild', 'qa-pull-deploy', tmppath.get_path(),
this.repo.get_path(), osname, targetName, targetRevision],
cancellable, { logInitiation: true });
}
- GSystem.file_linkcopy(latestPath, imageDir.get_child(latestPath.get_basename()),
- Gio.FileCopyFlags.OVERWRITE, cancellable);
+ GSystem.file_rename(newImageDir, this.imageDir.get_child(newImageDir.get_basename()),
+ cancellable);
+
+ let tmpLinkPath = Gio.File.new_for_path(this.imageDir, 'current-new.tmp');
+ GSystem.shutil_rm_rf(tmpLinkPath, cancellable);
+ tmpLinkPath.make_symbolic_link(newImageDir.get_basename(), cancellable);
+ let currentInfo = currentImageLink.query_info('standard::symlink-target', Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
+ let newPreviousTmppath = this.imageDir.get_child('previous-new.tmp');
+ GSystem.shutil_rm_rf(newPreviousTmppath, cancellable);
+ let currentLinkTarget = currentInfo.get_symlink_target();
+ newPreviousTmppath.make_symbolic_link(currentLinkTarget, cancellable);
+ GSystem.file_rename(newPreviousTmppath, previousImageLink);
+ GSystem.file_rename(tmpLinkPath, currentImageLink);
},
- _diskPathForTarget: function(targetName, isSnap) {
+ _diskNameForTarget: function(targetName, isSnap) {
let squashedName = targetName.replace(/\//g, '_');
let suffix;
if (isSnap) {
@@ -107,6 +128,6 @@ const BuildDisks = new Lang.Class({
} else {
suffix = '-disk.qcow2';
}
- return this.imageDir.get_child(this.prefix + '-' + squashedName + suffix);
+ return this.prefix + '-' + squashedName + suffix;
}
});
diff --git a/src/ostbuild/js/tasks/task-resolve.js b/src/ostbuild/js/tasks/task-resolve.js
new file mode 100644
index 0000000..1080b8a
--- /dev/null
+++ b/src/ostbuild/js/tasks/task-resolve.js
@@ -0,0 +1,84 @@
+// Copyright (C) 2011 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const JsonDB = imports.jsondb;
+const Builtin = imports.builtin;
+const Task = imports.task;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const TaskResolve = new Lang.Class({
+ Name: "TaskResolve",
+ Extends: Task.TaskDef,
+
+ TaskPattern: [/resolve\/(.*?)$/, 'prefix'],
+
+ DefaultParameters: {fetchAll: false,
+ fetchComponents: [],
+ timeoutSec: 10},
+
+ execute: function(cancellable) {
+ this.prefix = this.vars['prefix'];
+ let manifest = this.config.getGlobal('manifest');
+ let manifestPath = Gio.File.new_for_path(manifest);
+ let data = JsonUtil.loadJson(manifestPath, cancellable);
+ this._snapshot = new Snapshot.Snapshot(data, manifestPath, { prepareResolve: true });
+
+ let gitMirrorArgs = ['ostbuild', 'git-mirror', '--timeout-sec=' + this.parameters.timeoutSec,
+ '--manifest=' + manifest];
+ if (this.parameters.fetchAll || this.parameters.fetchComponents.length > 0) {
+ gitMirrorArgs.push('--fetch');
+ gitMirrorArgs.push('-k');
+ gitMirrorArgs.push.apply(gitMirrorArgs, this.parameters.fetchComponents);
+ }
+ ProcUtil.runSync(gitMirrorArgs, cancellable, { logInitiation: true });
+
+ let componentNames = this._snapshot.getAllComponentNames();
+ for (let i = 0; i < componentNames.length; i++) {
+ let component = this._snapshot.getComponent(componentNames[i]);
+ let src = component['src'];
+ let [keytype, uri] = Vcs.parseSrcKey(src);
+ let branch = component['branch'];
+ let tag = component['tag'];
+ let branchOrTag = branch || tag;
+ let mirrordir = Vcs.ensureVcsMirror(this.mirrordir, keytype, uri, branchOrTag, cancellable);
+ let revision = Vcs.describeVersion(mirrordir, branchOrTag);
+ component['revision'] = revision;
+ }
+
+ let snapshotdir = this.workdir.get_child('snapshots');
+ this._src_db = new JsonDB.JsonDB(snapshotdir.get_child(this.prefix));
+ let [path, modified] = this._src_db.store(this._snapshot.data, cancellable);
+ if (modified) {
+ print("New source snapshot: " + path.get_path());
+ } else {
+ print("Source snapshot unchanged: " + path.get_path());
+ }
+ }
+});
diff --git a/src/ostbuild/js/vcs.js b/src/ostbuild/js/vcs.js
index e2f82ad..3d324c2 100644
--- a/src/ostbuild/js/vcs.js
+++ b/src/ostbuild/js/vcs.js
@@ -173,22 +173,40 @@ function _listSubmodules(mirrordir, mirror, keytype, uri, branch, cancellable) {
function ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
params) {
- params = Params.parse(params, {fetch: false,
- fetchKeepGoing: false});
+ params = Params.parse(params, { fetch: false,
+ fetchKeepGoing: false,
+ timeoutSec: 0 });
+ let fetch = params.fetch;
let mirror = getMirrordir(mirrordir, keytype, uri);
let tmpMirror = mirror.get_parent().get_child(mirror.get_basename() + '.tmp');
let didUpdate = false;
let lastFetchPath = getLastfetchPath(mirrordir, keytype, uri, branch);
let lastFetchContents = null;
- if (lastFetchPath.query_exists(cancellable)) {
+ let currentTime = GLib.DateTime.new_now_utc();
+ let lastFetchContents = null;
+ let lastFetchInfo = null;
+ try {
+ lastFetchInfo = lastFetchPath.query_info('time::modified', Gio.FileQueryInfoFlags.NONE, cancellable);
+ } catch (e) {
+ if (!e.matches(Gio.IOErrorEnum, Gio.IOErrorEnum.NOT_FOUND))
+ throw e;
+ }
+ if (lastFetchInfo != null) {
lastFetchContents = GSystem.file_load_contents_utf8(lastFetchPath, cancellable).replace(/[ \n]/g, '');
+ if (params.timeoutSec > 0) {
+ let lastFetchTime = GLib.DateTime.new_from_unix_local(lastFetchInfo.get_attribute_uint64('time::modified'));
+ let diff = currentTime.difference(lastFetchTime) / 1000 / 1000;
+ if (diff < params.timeoutSec) {
+ fetch = false;
+ }
+ }
}
GSystem.shutil_rm_rf(tmpMirror, cancellable);
if (!mirror.query_exists(cancellable)) {
ProcUtil.runSync(['git', 'clone', '--mirror', uri, tmpMirror.get_path()], cancellable);
ProcUtil.runSync(['git', 'config', 'gc.auto', '0'], cancellable, {cwd: tmpMirror});
GSystem.file_rename(tmpMirror, mirror, cancellable);
- } else if (params.fetch) {
+ } else if (fetch) {
try {
ProcUtil.runSync(['git', 'fetch'], cancellable, {cwd:mirror});
} catch (e) {
@@ -210,17 +228,24 @@ function ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
});
}
- if (changed) {
+ if (changed || (fetch && params.timeoutSec > 0)) {
lastFetchPath.replace_contents(currentVcsVersion, null, false, 0, cancellable);
}
return mirror;
}
+function uncacheRepository(mirrordir, keytype, uri, branch, cancellable) {
+ let lastFetchPath = getLastfetchPath(mirrordir, keytype, uri, branch);
+ GSystem.shutil_rm_rf(lastFetchPath, cancellable);
+}
+
function fetch(mirrordir, keytype, uri, branch, cancellable, params) {
- params = Params.parse(params, {keepGoing: false});
+ params = Params.parse(params, {keepGoing: false, timeoutSec: 0});
ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
- {fetch:true, fetchKeepGoing: params.keepGoing});
+ { fetch:true,
+ fetchKeepGoing: params.keepGoing,
+ timeoutSec: params.timeoutSec });
}
function describeVersion(dirpath, branch) {
diff --git a/src/ostbuild/ostbuild.in b/src/ostbuild/ostbuild.in
index 01d75bc..258d9d0 100755
--- a/src/ostbuild/ostbuild.in
+++ b/src/ostbuild/ostbuild.in
@@ -25,4 +25,4 @@ export GIO_USE_VFS=local
export OSTBUILD_DATADIR= pkgdatadir@
export OSTBUILD_LIBDIR= pkglibdir@
-exec gjs -I "${jsdir}" "${jsdir}/main.js" "$@"
+exec $OSTBUILD_GDB gjs -I "${jsdir}" "${jsdir}/main.js" "$@"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
