[gnome-ostree] Rewrite tasks system
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree] Rewrite tasks system
- Date: Sun, 10 Feb 2013 18:42:41 +0000 (UTC)
commit c02802a6a7c6d0d0fb2a6c1ddbe886c926120b98
Author: Colin Walters <walters verbum org>
Date: Thu Jan 24 09:13:51 2013 -0500
Rewrite tasks system
Merge SubTask and DynTask for now - drop the dependencies, clean
things up a bit. Move resolve, build, builddisks to be tasks.
The autobuilder now is updated to use this.
Add a new bdiff task.
There's a new "ostbuild make" builtin that runs tasks. The benefit of
this when you do a build directly, like:
$ ostbuild make build/gnomeos-3.8
It's consistently run in a tempdir, logged etc., in the same way that
the autobuilder does it.
Makefile-ostbuild.am | 16 +-
qa/repoweb/index.html | 121 ++++---
qa/repoweb/repoweb.js | 101 ++----
src/libgsystem | 2 +-
src/ostbuild/js/buildutil.js | 81 ++---
src/ostbuild/js/builtins/autobuilder.js | 316 +++++------------
src/ostbuild/js/builtins/build_disks.js | 112 ------
src/ostbuild/js/builtins/git_mirror.js | 28 +-
src/ostbuild/js/builtins/make.js | 86 +++++
src/ostbuild/js/builtins/resolve.js | 140 -------
src/ostbuild/js/builtins/run_task.js | 55 +++
src/ostbuild/js/dyntask.js | 244 -------------
src/ostbuild/js/jsondb.js | 12 +
src/ostbuild/js/jsonutil.js | 38 ++
src/ostbuild/js/jsutil.js | 23 ++
src/ostbuild/js/main.js | 2 +
src/ostbuild/js/procutil.js | 2 +-
src/ostbuild/js/snapshot.js | 75 ++++-
src/ostbuild/js/subtask.js | 181 ----------
src/ostbuild/js/task.js | 381 ++++++++++++++++++++
src/ostbuild/js/tasks/task-bdiff.js | 155 ++++++++
.../js/{builtins/build.js => tasks/task-build.js} | 109 +++----
src/ostbuild/js/tasks/task-builddisks.js | 141 ++++++++
src/ostbuild/js/tasks/task-checksum.js | 123 -------
src/ostbuild/js/tasks/task-resolve.js | 84 +++++
.../qa_smoketest.js => tasks/task-smoketest.js} | 126 ++++---
src/ostbuild/js/vcs.js | 39 ++-
src/ostbuild/ostbuild.in | 2 +-
28 files changed, 1452 insertions(+), 1343 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index ef0ef1f..24c73ce 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -43,9 +43,10 @@ jsostbuild_DATA= \
src/ostbuild/js/buildutil.js \
src/ostbuild/js/builtin.js \
src/ostbuild/js/config.js \
- src/ostbuild/js/dyntask.js \
+ src/ostbuild/js/task.js \
src/ostbuild/js/jsondb.js \
src/ostbuild/js/jsonutil.js \
+ src/ostbuild/js/jsutil.js \
src/ostbuild/js/main.js \
src/ostbuild/js/libqa.js \
src/ostbuild/js/guestfish.js \
@@ -53,28 +54,29 @@ jsostbuild_DATA= \
src/ostbuild/js/procutil.js \
src/ostbuild/js/snapshot.js \
src/ostbuild/js/streamutil.js \
- src/ostbuild/js/subtask.js \
src/ostbuild/js/vcs.js \
$(NULL)
jsostbuiltinsdir=$(jsostbuilddir)/builtins
jsostbuiltins_DATA= \
src/ostbuild/js/builtins/autobuilder.js \
- src/ostbuild/js/builtins/build.js \
- src/ostbuild/js/builtins/build_disks.js \
src/ostbuild/js/builtins/checkout.js \
src/ostbuild/js/builtins/git_mirror.js \
+ src/ostbuild/js/builtins/make.js \
src/ostbuild/js/builtins/qa_make_disk.js \
src/ostbuild/js/builtins/qa_pull_deploy.js \
- src/ostbuild/js/builtins/qa_smoketest.js \
src/ostbuild/js/builtins/prefix.js \
- src/ostbuild/js/builtins/resolve.js \
+ src/ostbuild/js/builtins/run_task.js \
src/ostbuild/js/builtins/shell.js \
$(NULL)
jsosttasksdir=$(jsostbuilddir)/tasks
jsosttasks_DATA= \
- src/ostbuild/js/tasks/task-checksum.js \
+ src/ostbuild/js/tasks/task-build.js \
+ src/ostbuild/js/tasks/task-resolve.js \
+ src/ostbuild/js/tasks/task-bdiff.js \
+ src/ostbuild/js/tasks/task-builddisks.js \
+ src/ostbuild/js/tasks/task-smoketest.js \
$(NULL)
endif
diff --git a/qa/repoweb/index.html b/qa/repoweb/index.html
index 30765e5..5e9722e 100644
--- a/qa/repoweb/index.html
+++ b/qa/repoweb/index.html
@@ -1,63 +1,66 @@
<!DOCTYPE html>
<html>
- <head>
- <meta charset="utf-8" />
- <meta name="viewport" content="width=device-width, initial-scale=1" />
- <title>
- </title>
- <link rel="stylesheet" href="jquery.mobile-1.2.0.css" />
- <link rel="stylesheet" href="repoweb.css" />
- <style>
- /* App custom styles */
- </style>
- <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js";>
- </script>
- <script src="https://ajax.aspnetcdn.com/ajax/jquery.mobile/1.2.0/jquery.mobile-1.2.0.min.js";>
- </script>
- <script src="repoweb.js">
- </script>
- </head>
- <body onload="$(document).ready(function(){repoweb_index_init();})">
- <!-- Home -->
- <div data-role="page" data-theme="a" id="page1">
- <div data-theme="" data-role="header">
- <h3>
- GNOME-OSTree build server
- </h3>
- <div data-role="navbar" data-iconpos="top">
- <ul>
- <li>
- <a href="#page1" data-transition="fade" data-theme="" data-icon="">
- Home
- </a>
- </li>
- <li>
- <a href="work/tasks" rel="external">
- Build logs
- </a>
- </li>
- <li>
- <a href="work/src" rel="external">
- Source code
- </a>
- </li>
- </ul>
- </div>
- </div>
- <div data-role="content">
- <div class="buildstatus" id="buildstatus">
- </div>
-<p>This is the <a href="https://live.gnome.org/OSTree/GnomeOSTree"; class="ui-link">GNOME-OSTree</a> build server. It builds from the <a href="http://git.gnome.org/browse/gnome-ostree"; class="ui-link">gnome-ostree</a> git module.</p>
- <ul data-role="listview" data-divider-theme="a" data-inset="true" id="build-summary">
- </ul>
- </div>
- <div class="footer">
- <p>Optimized for standards. Powered by <a href="http://jquerymobile.com"; data-transition="fade">jquery-mobile</a>.</p>
- </div>
+ <head>
+ <meta charset="utf-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
+ <title>
+ </title>
+ <link rel="stylesheet" href="jquery.mobile-1.2.0.css" />
+ <link rel="stylesheet" href="repoweb.css" />
+ <style>
+ /* App custom styles */
+ </style>
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js";>
+ </script>
+ <script src="https://ajax.aspnetcdn.com/ajax/jquery.mobile/1.2.0/jquery.mobile-1.2.0.min.js";>
+ </script>
+ <script src="repoweb.js">
+ </script>
+ </head>
+ <body onload="$(document).ready(function(){repowebIndexInit();})">
+ <!-- Home -->
+ <div data-role="page" data-theme="a" id="page1">
+ <div data-theme="" data-role="header">
+ <h3>
+ GNOME-OSTree build server
+ </h3>
+ <div data-role="navbar" data-iconpos="top">
+ <ul>
+ <li>
+ <a href="#page1" data-transition="fade" data-theme="" data-icon="">
+ Home
+ </a>
+ </li>
+ <li>
+ <a href="work/tasks" rel="external">
+ Build logs
+ </a>
+ </li>
+ <li>
+ <a href="work/src" rel="external">
+ Source code
+ </a>
+ </li>
+ </ul>
+ </div>
+ </div>
+ <div data-role="content">
+ <div class="content-primary">
+ <div class="buildstatus" id="build-icon">
+ </div>
+ <p>This is the <a href="https://live.gnome.org/OSTree/GnomeOSTree"; class="ui-link">GNOME-OSTree</a> build server. It builds from the <a href="http://git.gnome.org/browse/gnome-ostree"; class="ui-link">gnome-ostree</a> git module.</p>
+ <p>Current build:
+ <span id="build-meta"></span>
+ </p>
</div>
-
- <script>
- //App custom javascript
- </script>
- </body>
+ </div>
+ <div class="footer">
+ <p>Optimized for standards. Powered by <a href="http://jquerymobile.com"; data-transition="fade">jquery-mobile</a>.</p>
+ </div>
+ </div>
+
+ <script>
+ //App custom javascript
+ </script>
+ </body>
</html>
diff --git a/qa/repoweb/repoweb.js b/qa/repoweb/repoweb.js
index a3bacf9..1b96276 100644
--- a/qa/repoweb/repoweb.js
+++ b/qa/repoweb/repoweb.js
@@ -1,5 +1,7 @@
// -*- indent-tabs-mode: nil -*-
+const DEFAULT_PREFIX = 'gnomeos-3.8';
+
function htmlescape(str) {
var pre = document.createElement('pre');
var text = document.createTextNode(str);
@@ -26,22 +28,19 @@ function get_page_arg(key) {
}
var repoDataSignal = {};
-var repoData = null;
+var currentBuildMeta = null;
var prefix = null;
-function repoweb_on_data_loaded(data) {
- console.log("data loaded");
- repoData = data;
- prefix = repoData['prefix'];
- $(repoDataSignal).trigger("loaded");
-}
-
-function repoweb_init() {
- var id = get_page_arg("prefix");
- if (id == null)
- id = "default";
- var url = "work/autobuilder-" + id + ".json";
- $.getJSON(url, repoweb_on_data_loaded);
+function repowebInit() {
+ prefix = get_page_arg("prefix");
+ if (prefix == null)
+ prefix = DEFAULT_PREFIX;
+ var url;
+ url = "work/tasks/build/" + prefix + "/current/meta.json";
+ $.getJSON(url, function(data) {
+ currentBuildMeta = data;
+ $(repoDataSignal).trigger("current-buildmeta-loaded");
+ });
}
function timeago(d, now) {
@@ -61,22 +60,6 @@ function timeago(d, now) {
}
}
-function buildDiffComponentAppend(container, description, a) {
- var additional = 0;
- if (a.length > 10) {
- a = a.slice(0, 10);
- additional = a.length - 10;
- }
- var p = document.createElement('p');
- container.appendChild(p);
- p.appendChild(document.createTextNode(description + ": " + a.join(", ")));
- if (additional > 0) {
- var b = document.createElement('b');
- p.appendChild(b);
- b.appendChild.document.createTextNode(" and " + additional + " more");
- }
-}
-
function buildDiffAppend(container, buildDiff) {
if (!buildDiff)
return document.createTextNode("No changes or new build");
@@ -139,41 +122,31 @@ function renderBuild(container, build) {
}
-function updateResolve() {
- $("#resolve-summary").empty();
- var summary = $("#resolve-summary").get(0);
-
- var div = document.createElement('div');
- summary.appendChild(div);
- div.appendChild(document.createTextNode("Current version: "));
- var a = document.createElement('a');
- div.appendChild(a);
- a.setAttribute('href', 'work/snapshots/' + repoData['version-path']);
- a.setAttribute('rel', 'external');
- a.appendChild(document.createTextNode(repoData['version']));
-}
-
-function repoweb_index_init() {
- repoweb_init();
- $(repoDataSignal).on("loaded", function () {
-
- var buildSummary = $("#build-summary").get(0);
- var buildData = repoData.build;
- for (var i = buildData.length - 1; i >= 0; i--) {
- var build = buildData[i];
- renderBuild(buildSummary, build);
- }
- if (buildData.length > 0) {
- var build = buildData[0];
- $("#buildstatus").removeClass("buildstatus-happy");
- $("#buildstatus").removeClass("buildstatus-sad");
- if (build['state'] == 'failed')
- $("#buildstatus").addClass("buildstatus-sad");
- else
- $("#buildstatus").addClass("buildstatus-happy");
+function repowebIndexInit() {
+ repowebInit();
+ $(repoDataSignal).on("current-buildmeta-loaded", function () {
+ var buildMetaNode = $("#build-meta").get(0);
+
+ $(buildMetaNode).empty();
+ var ref = 'work/tasks/build/' + prefix;
+ if (currentBuildMeta.success)
+ ref += '/successful';
+ else
+ ref += '/failed';
+ ref += '/' + currentBuildMeta.taskVersion;
+ var a = document.createElement('a');
+ a.setAttribute('href', ref);
+ a.setAttribute('rel', 'external');
+ a.appendChild(document.createTextNode(currentBuildMeta.taskVersion));
+ buildMetaNode.appendChild(a);
+ buildMetaNode.appendChild(document.createTextNode(': ' + (currentBuildMeta.success ? "success" : "failed ")));
+
+ $("#build-icon").removeClass("buildstatus-happy");
+ $("#build-icon").removeClass("buildstatus-sad");
+ if (currentBuildMeta.success) {
+ $("#build-icon").addClass("buildstatus-happy");
} else {
- $("#buildstatus").addClass("buildstatus-happy");
+ $("#build-icon").addClass("buildstatus-sad");
}
- $(buildSummary).listview('refresh');
});
}
diff --git a/src/libgsystem b/src/libgsystem
index dd1b303..0258d06 160000
--- a/src/libgsystem
+++ b/src/libgsystem
@@ -1 +1 @@
-Subproject commit dd1b3032b4cd175c2a9b2c611a62525454bad772
+Subproject commit 0258d06e5342bc4617f88594324eed43269e133e
diff --git a/src/ostbuild/js/buildutil.js b/src/ostbuild/js/buildutil.js
index 0f3d1c7..5bcfcb4 100644
--- a/src/ostbuild/js/buildutil.js
+++ b/src/ostbuild/js/buildutil.js
@@ -19,6 +19,8 @@ const GLib = imports.gi.GLib;
const Gio = imports.gi.Gio;
const Lang = imports.lang;
+const GSystem = imports.gi.GSystem;
+
const BUILD_ENV = {
'HOME' : '/',
'HOSTNAME' : 'ostbuild',
@@ -42,50 +44,7 @@ function parseSrcKey(srckey) {
return [keytype, uri];
}
-function resolveComponent(manifest, componentMeta) {
- let result = {};
- Lang.copyProperties(componentMeta, result);
- let origSrc = componentMeta['src'];
-
- let didExpand = false;
- let vcsConfig = manifest['vcsconfig'];
- for (let vcsprefix in vcsConfig) {
- let expansion = vcsConfig[vcsprefix];
- let prefix = vcsprefix + ':';
- if (origSrc.indexOf(prefix) == 0) {
- result['src'] = expansion + origSrc.substr(prefix.length);
- didExpand = true;
- break;
- }
- }
-
- let name = componentMeta['name'];
- let src, idx, name;
- if (name == undefined) {
- if (didExpand) {
- src = origSrc;
- idx = src.lastIndexOf(':');
- name = src.substr(idx+1);
- } else {
- src = result['src'];
- idx = src.lastIndexOf('/');
- name = src.substr(idx+1);
- }
- let i = name.lastIndexOf('.git');
- if (i != -1 && i == name.length - 4) {
- name = name.substr(0, name.length - 4);
- }
- name = name.replace(/\//g, '-');
- result['name'] = name;
- }
-
- let branchOrTag = result['branch'] || result['tag'];
- if (!branchOrTag) {
- result['branch'] = 'master';
- }
- return result;
-}
function getPatchPathsForComponent(patchdir, component) {
let patches = component['patches'];
@@ -127,3 +86,39 @@ function getBaseUserChrootArgs() {
let path = findUserChrootPath();
return [path.get_path(), '--unshare-pid', '--unshare-ipc', '--unshare-net'];
}
+
+function compareVersions(a, b) {
+ let adot = a.indexOf('.');
+ while (adot != -1) {
+ let bdot = b.indexOf('.');
+ if (bdot == -1)
+ return 1;
+ let aSub = parseInt(a.substr(0, adot));
+ let bSub = parseInt(b.substr(0, bdot));
+ if (aSub > bSub)
+ return 1;
+ else if (aSub < bSub)
+ return -1;
+ a = a.substr(adot + 1);
+ b = b.substr(bdot + 1);
+ adot = a.indexOf('.');
+ }
+ if (b.indexOf('.') != -1)
+ return -1;
+ let aSub = parseInt(a);
+ let bSub = parseInt(b);
+ if (aSub > bSub)
+ return 1;
+ else if (aSub < bSub)
+ return -1;
+ return 0;
+}
+
+function atomicSymlinkSwap(linkPath, newTarget, cancellable) {
+ let parent = linkPath.get_parent();
+ let tmpLinkPath = parent.get_child('current-new.tmp');
+ GSystem.shutil_rm_rf(tmpLinkPath, cancellable);
+ let relpath = parent.get_relative_path(newTarget);
+ tmpLinkPath.make_symbolic_link(relpath, cancellable);
+ GSystem.file_rename(tmpLinkPath, linkPath, cancellable);
+}
diff --git a/src/ostbuild/js/builtins/autobuilder.js b/src/ostbuild/js/builtins/autobuilder.js
index 17268c8..b437175 100644
--- a/src/ostbuild/js/builtins/autobuilder.js
+++ b/src/ostbuild/js/builtins/autobuilder.js
@@ -23,7 +23,7 @@ const Format = imports.format;
const GSystem = imports.gi.GSystem;
const Builtin = imports.builtin;
-const SubTask = imports.subtask;
+const Task = imports.task;
const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const JsonUtil = imports.jsonutil;
@@ -54,14 +54,12 @@ const Autobuilder = new Lang.Class({
this._stages = ['resolve', 'build', 'builddisks', 'smoke'];
- this._build_needed = true;
- this._do_builddisks = false;
- this._do_qa = false;
- this._full_resolve_needed = true;
- this._queued_force_resolve = [];
- this._resolve_timeout = 0;
- this._source_snapshot_path = null;
- this._prev_source_snapshot_path = null;
+ this._buildNeeded = true;
+ this._fullResolveNeeded = true;
+ this._resolveNeeded = false;
+ this._resolveTimeout = 0;
+ this._sourceSnapshotPath = null;
+ this._prevSourceSnapshotPath = null;
},
execute: function(args, loop, cancellable) {
@@ -69,14 +67,17 @@ const Autobuilder = new Lang.Class({
this._autoupdate_self = args.autoupdate_self;
if (!args.stage)
- args.stage = 'smoke';
+ args.stage = 'build';
this._stageIndex = this._stages.indexOf(args.stage);
if (this._stageIndex < 0)
throw new Error("Unknown stage " + args.stage);
this._do_builddisks = this._stageIndex >= this._stages.indexOf('builddisks');
this._do_smoke = this._stageIndex >= this._stages.indexOf('smoke');
- this._status_path = this.workdir.get_child('autobuilder-' + this.prefix + '.json');
+ this._resolveTaskName = 'resolve/' + this.prefix;
+ this._buildTaskName = 'build/' + this.prefix;
+ this._bdiffTaskName = 'bdiff/' + this.prefix;
+
this._manifestPath = Gio.File.new_for_path('manifest.json');
this._ownId = Gio.DBus.session.own_name('org.gnome.OSTreeBuild', Gio.BusNameOwnerFlags.NONE,
@@ -89,35 +90,42 @@ const Autobuilder = new Lang.Class({
this._snapshot_dir = this.workdir.get_child('snapshots').get_child(this.prefix);
this._src_db = new JsonDB.JsonDB(this._snapshot_dir);
- let taskdir = this.workdir.get_child('tasks');
- this._resolve_taskset = new SubTask.TaskSet(taskdir.get_child(this.prefix + '-resolve'));
- this._build_taskset = new SubTask.TaskSet(taskdir.get_child(this.prefix + '-build'));
- this._builddisks_taskset = new SubTask.TaskSet(taskdir.get_child(this.prefix + '-build-disks'));
- this._smoke_taskset = new SubTask.TaskSet(taskdir.get_child(this.prefix + '-smoke'));
-
- this._source_snapshot_path = this._src_db.getLatestPath();
+ this._taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
+ { onEmpty: Lang.bind(this, this._onTasksComplete) });
+ this._taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
- this._status_path = this.workdir.get_child('autobuilder-' + this.prefix + '.json');
+ this._sourceSnapshotPath = this._src_db.getLatestPath();
- this._resolve_timeout = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT,
- 60 * 10, Lang.bind(this, this._fetchAll));
- this._fetchAll();
- if (this._source_snapshot_path != null)
- this._run_build();
+ this._resolveTimeout = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT,
+ 60 * 10, Lang.bind(this, this._triggerFullResolve));
+ this._runResolve();
+ if (this._sourceSnapshotPath != null)
+ this._runBuild();
this._updateStatus();
loop.run();
},
+ _onTasksComplete: function() {
+ },
+
+ _onTaskCompleted: function(taskmaster, task, success, error) {
+ if (task.name == this._resolveTaskName) {
+ this._onResolveExited(task, success, error);
+ } else if (task.name == this._buildTaskName) {
+ this._onBuildExited(task, success, error);
+ }
+ this._updateStatus();
+ },
+
_updateStatus: function() {
let newStatus = "";
- if (this._resolve_taskset.isRunning())
- newStatus += "[resolving] ";
- if (this._build_taskset.isRunning())
- newStatus += " [building] ";
- if (this._builddisks_taskset.isRunning())
- newStatus += " [disks] ";
+ let taskstateList = this._taskmaster.getTaskState();
+ for (let i = 0; i < taskstateList.length; i++) {
+ let taskstate = taskstateList[i];
+ newStatus += (taskstate.task.name + " ");
+ }
if (newStatus == "")
newStatus = "[idle]";
if (newStatus != this._status) {
@@ -125,8 +133,6 @@ const Autobuilder = new Lang.Class({
print(this._status);
this._impl.emit_property_changed('Status', new GLib.Variant("s", this._status));
}
-
- this._writeStatusFile();
},
get Status() {
@@ -135,234 +141,92 @@ const Autobuilder = new Lang.Class({
queueResolve: function(srcUrls) {
let matchingComponents = [];
- let snapshotData = this._src_db.loadFromPath(this._source_snapshot_path, null);
- let snapshot = new Snapshot.Snapshot(snapshotData, this._source_snapshot_path);
+ let snapshotData = this._src_db.loadFromPath(this._sourceSnapshotPath, null);
+ let snapshot = new Snapshot.Snapshot(snapshotData, this._sourceSnapshotPath);
+ let matched = false;
for (let i = 0; i < srcUrls.length; i++) {
let matches = snapshot.getMatchingSrc(srcUrls[i]);
- for (let j = 0; j < matches.length; j++)
- matchingComponents.push(matches[j]['name']);
- }
- if (matchingComponents.length > 0) {
- this._queued_force_resolve.push.apply(this._queued_force_resolve, matchingComponents);
- print("queued resolves: " + matchingComponents.join(' '));
- if (!this._resolve_taskset.isRunning())
- this._fetch();
- } else {
- print("Ignored fetch requests for unknown URLs: " + srcUrls.join(','));
+ for (let j = 0; j < matches.length; j++) {
+ this._queuedForceResolve.push.apply(this._queuedForceResolve, matches[i]['name']);
+ matched = true;
+ }
}
+ if (matched)
+ this._resolveNeeded = true;
+ this._runResolve();
},
- _fetchAll: function() {
- this._full_resolve_needed = true;
- if (!this._resolve_taskset.isRunning())
- this._fetch();
+ _triggerFullResolve: function() {
+ this._fullResolveNeeded = true;
+ this._runResolve();
return true;
},
- _fetch: function() {
+ _runResolve: function() {
let cancellable = null;
+
+ if (!(this._resolveNeeded || this._fullResolveNeeded))
+ return;
+
+ if (this._taskmaster.isTaskQueued(this._resolveTaskName))
+ return;
if (this._autoupdate_self)
ProcUtil.runSync(['git', 'pull', '-r'], cancellable)
- let args = ['ostbuild', 'resolve', '--manifest=manifest.json',
- '--fetch', '--fetch-keep-going'];
- let isFull;
- if (this._full_resolve_needed) {
- this._full_resolve_needed = false;
- isFull = true;
- } else if (this._queued_force_resolve.length > 0) {
- args.push.apply(args, this._queued_force_resolve);
- isFull = false;
+ if (this._fullResolveNeeded) {
+ this._fullResolveNeeded = false;
+ this._taskmaster.pushTask(this._resolveTaskName,
+ { fetchAll: true });
} else {
- throw new Error("_fetch() when not needed");
+ this._taskmaster.pushTask(this._resolveTaskName,
+ { fetchComponents: this._queuedForceResolve });
}
- this._queued_force_resolve = [];
- let context = new GSystem.SubprocessContext({ argv: args });
- let workdir = this._resolve_taskset.prepare();
- let tmpManifest = workdir.get_child(this._manifestPath.get_basename());
- GSystem.file_linkcopy(this._manifestPath, tmpManifest, Gio.FileCopyFlags.OVERWRITE, cancellable);
- let t = this._resolve_taskset.start(context,
- cancellable,
- Lang.bind(this, this._onResolveExited));
- print(Format.vprintf("Resolve task %s started (%s)", [t.versionstr, isFull ? "full" : "incremental"]));
+ this._queuedForceResolve = [];
this._updateStatus();
-
- return false;
},
_onResolveExited: function(resolveTask, success, msg) {
print(Format.vprintf("resolve exited; success=%s msg=%s", [success, msg]))
- this._prev_source_snapshot_path = this._source_snapshot_path;
- this._source_snapshot_path = this._src_db.getLatestPath();
- let changed = (this._prev_source_snapshot_path == null ||
- !this._prev_source_snapshot_path.equal(this._source_snapshot_path));
+ this._prevSourceSnapshotPath = this._sourceSnapshotPath;
+ this._sourceSnapshotPath = this._src_db.getLatestPath();
+ let changed = (this._prevSourceSnapshotPath == null ||
+ !this._prevSourceSnapshotPath.equal(this._sourceSnapshotPath));
if (changed)
- print(Format.vprintf("New version is %s", [this._source_snapshot_path.get_path()]))
- if (!this._build_needed)
- this._build_needed = changed;
- if (this._build_needed && !this._build_taskset.isRunning())
- this._run_build();
-
- if (this._full_resolve_needed || this._queued_force_resolve.length > 0) {
- this._fetch();
- }
-
+ print(Format.vprintf("New version is %s", [this._sourceSnapshotPath.get_path()]))
+ if (!this._buildNeeded)
+ this._buildNeeded = changed;
+ this._runBuild();
+ this._runResolve();
this._updateStatus();
},
-
- _run_build: function() {
- let cancellable = null;
- if (this._build_taskset.isRunning()) throw new Error();
- if (!this._build_needed) throw new Error();
- this._build_needed = false;
-
- let snapshotName = this._source_snapshot_path.get_basename();
-
- let workdir = this._build_taskset.prepare();
- let tmpSnapshotPath = workdir.get_child(snapshotName);
- GSystem.file_linkcopy(this._source_snapshot_path, tmpSnapshotPath,
- Gio.FileCopyFlags.OVERWRITE, cancellable);
-
- let version = this._src_db.parseVersionStr(this._source_snapshot_path.get_basename());
- let meta = {'version': version,
- 'version-path': this._snapshot_dir.get_relative_path(this._source_snapshot_path)};
- let metaPath = workdir.get_child('meta.json');
- JsonUtil.writeJsonFileAtomic(metaPath, meta, cancellable);
-
- let args = ['ostbuild', 'build', '--snapshot=' + snapshotName];
-
- let context = new GSystem.SubprocessContext({ argv: args });
- let task = this._build_taskset.start(context,
- cancellable,
- Lang.bind(this, this._onBuildExited));
- print(Format.vprintf("Build task %s started", [task.versionstr]));
-
- this._updateStatus();
+ _onBuildExited: function(buildTaskset, success, msg) {
+ print(Format.vprintf("build exited; success=%s msg=%s", [success, msg]))
+ if (this._buildNeeded)
+ this._runBuild()
+
+ this._updateStatus();
},
-
- _run_builddisks: function() {
+
+ _runBuild: function() {
let cancellable = null;
-
- if (!this._do_builddisks || this._builddisks_taskset.isRunning())
+ if (this._taskmaster.isTaskQueued(this._buildTaskName))
return;
-
- let args = ['ostbuild', 'build-disks'];
-
- let context = new GSystem.SubprocessContext({ argv: args });
- let task = this._builddisks_taskset.start(context,
- cancellable,
- Lang.bind(this, this._onBuildDisksExited));
- print(Format.vprintf("Builddisks task %s started", [task.versionstr]));
-
- this._updateStatus();
- },
-
- _run_smoke: function() {
- let cancellable = null;
-
- if (!this._do_smoke || this._smoke_taskset.isRunning())
+ if (!this._buildNeeded)
return;
- let args = ['ostbuild', 'qa-smoketest'];
-
- let context = new GSystem.SubprocessContext({ argv: args });
- let task = this._smoke_taskset.start(context,
- cancellable,
- Lang.bind(this, this._onSmokeExited));
- print(Format.vprintf("Smoke task %s started", [task.versionstr]));
-
+ this._buildNeeded = false;
+ this._taskmaster.pushTask(this._buildTaskName);
this._updateStatus();
},
- _onBuildExited: function(buildTaskset, success, msg) {
- print(Format.vprintf("build exited; success=%s msg=%s", [success, msg]))
- if (this._build_needed)
- this._run_build()
- if (success)
- this._run_builddisks();
-
- this._updateStatus();
- },
-
- _onBuildDisksExited: function(buildTaskset, success, msg) {
- print(Format.vprintf("builddisks exited; success=%s msg=%s", [success, msg]))
- this._updateStatus();
-
- if (success)
- this._run_smoke();
+ _runBdiff: function() {
+ if (this._taskmaster.isTaskQueued(this._bdiffTaskName))
+ return;
+ this._taskmaster.pushTask(this._bdiffTaskName);
this._updateStatus();
- },
-
- _getBuildDiffForTask: function(task) {
- let cancellable = null;
- if (task.build_diff != undefined)
- return task.build_diff;
- let metaPath = task.path.get_child('meta.json');
- if (!metaPath.query_exists(null)) {
- task.build_diff = null;
- return task.build_diff;
- }
- let meta = JsonUtil.loadJson(metaPath, cancellable);
- let snapshotPath = this._snapshot_dir.get_child(meta['version-path']);
- let prevSnapshotPath = this._src_db.getPreviousPath(snapshotPath);
- if (prevSnapshotPath == null) {
- task.build_diff = null;
- } else {
- task.build_diff = Snapshot.snapshotDiff(this._src_db.loadFromPath(snapshotPath, cancellable),
- this._src_db.loadFromPath(prevSnapshotPath, cancellable));
- }
- return task.build_diff;
- },
-
- _buildHistoryToJson: function() {
- let cancellable = null;
- let history = this._build_taskset.getHistory();
- let l = history.length;
- let MAXITEMS = 5;
- let entries = [];
- for (let i = Math.max(l - MAXITEMS, 0); i >= 0 && i < l; i++) {
- let item = history[i];
- let data = {v: item.versionstr,
- state: item.state,
- timestamp: item.timestamp};
- entries.push(data);
- let metaPath = item.path.get_child('meta.json');
- if (metaPath.query_exists(cancellable)) {
- data['meta'] = JsonUtil.loadJson(metaPath, cancellable);
- }
- data['diff'] = this._getBuildDiffForTask(item);
- }
- return entries;
- },
-
- _writeStatusFile: function() {
- let cancellable = null;
- let status = {'prefix': this.prefix};
- if (this._source_snapshot_path != null) {
- let version = this._src_db.parseVersionStr(this._source_snapshot_path.get_basename());
- status['version'] = version;
- status['version-path'] = this._snapshot_dir.get_relative_path(this._source_snapshot_path);
- } else {
- status['version'] = '';
- }
-
- status['build'] = this._buildHistoryToJson();
-
- if (this._build_proc != null) {
- let buildHistory = this._build_taskset.getHistory();
- let activeBuild = buildHistory[buildHistory.length-1];
- let buildStatus = status['build'];
- let activeBuildJson = buildStatus[buildStatus.length-1];
- let statusPath = activeBuild.path.get_child('status.json');
- if (statusPath.query_exists(null)) {
- activeBuildJson['build-status'] = JsonUtil.loadJson(statusPath);
- }
- }
-
- JsonUtil.writeJsonFileAtomic(this._status_path, status, cancellable);
}
});
diff --git a/src/ostbuild/js/builtins/git_mirror.js b/src/ostbuild/js/builtins/git_mirror.js
index b1101c3..d1c05cb 100644
--- a/src/ostbuild/js/builtins/git_mirror.js
+++ b/src/ostbuild/js/builtins/git_mirror.js
@@ -41,6 +41,7 @@ const GitMirror = new Lang.Class({
this.parser.addArgument('--prefix');
this.parser.addArgument('--manifest');
this.parser.addArgument('--snapshot');
+ this.parser.addArgument('--timeout-sec', { help: "Cache fetch results for provided number of seconds" });
this.parser.addArgument('--fetch', {action:'storeTrue',
help:"Also do a git fetch for components"});
this.parser.addArgument(['-k', '--keep-going'], {action:'storeTrue',
@@ -51,17 +52,13 @@ const GitMirror = new Lang.Class({
execute: function(args, loop, cancellable) {
let parser = new ArgParse.ArgumentParser();
+ if (!args.timeout_sec)
+ args.timeout_sec = 0;
+
if (args.manifest != null) {
- let snapshotData = JsonUtil.loadJson(Gio.File.new_for_path(args.manifest), cancellable);
- let resolvedComponents = [];
- let components = snapshotData['components'];
- for (let i = 0; i < components.length; i++) {
- resolvedComponents.push(BuildUtil.resolveComponent(snapshotData, components[i]));
- }
- snapshotData['components'] = resolvedComponents;
- snapshotData['patches'] = BuildUtil.resolveComponent(snapshotData, snapshotData['patches']);
- snapshotData['base'] = BuildUtil.resolveComponent(snapshotData, snapshotData['base']);
- this._snapshot = new Snapshot.Snapshot(snapshotData, null);
+ let manifestPath = Gio.File.new_for_path(args.manifest)
+ let manifestData = JsonUtil.loadJson(manifestPath, cancellable);
+ this._snapshot = new Snapshot.Snapshot(manifestData, manifestPath, { prepareResolve: true });
} else {
this._initSnapshot(args.prefix, args.snapshot, cancellable);
}
@@ -74,18 +71,15 @@ const GitMirror = new Lang.Class({
}
componentNames.forEach(Lang.bind(this, function (name) {
- let component = this._snapshot.getComponent(name);
- let src = component['src']
- let [keytype, uri] = Vcs.parseSrcKey(src);
- let branch = component['branch'];
- let tag = component['tag'];
- let branchOrTag = branch || tag;
+ let [keytype, uri, branchOrTag] = this._snapshot.getVcsInfo(name);
if (!args.fetch) {
Vcs.ensureVcsMirror(this.mirrordir, keytype, uri, branchOrTag, cancellable);
} else {
print("Running git fetch for " + name);
- Vcs.fetch(this.mirrordir, keytype, uri, branchOrTag, cancellable, {keepGoing:args.keep_going});
+ Vcs.fetch(this.mirrordir, keytype, uri, branchOrTag, cancellable,
+ { keepGoing:args.keep_going,
+ timeoutSec: args.timeout_sec });
}
}));
}
diff --git a/src/ostbuild/js/builtins/make.js b/src/ostbuild/js/builtins/make.js
new file mode 100644
index 0000000..cab5084
--- /dev/null
+++ b/src/ostbuild/js/builtins/make.js
@@ -0,0 +1,86 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const Task = imports.task;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const Make = new Lang.Class({
+ Name: 'Make',
+ Extends: Builtin.Builtin,
+
+ DESCRIPTION: "Execute tasks",
+
+ _init: function() {
+ this.parent();
+ this.parser.addArgument('taskname');
+ this.parser.addArgument('parameters', { nargs: '*' });
+ },
+
+ execute: function(args, loop, cancellable) {
+ this._loop = loop;
+ this._failed = false;
+ let taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
+ { onEmpty: Lang.bind(this, this._onTasksComplete) });
+ this._taskmaster = taskmaster;
+ taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
+ let params = {};
+ for (let i = 0; i < args.parameters.length; i++) {
+ let param = args.parameters[i];
+ let idx = param.indexOf('=');
+ if (idx == -1)
+ throw new Error("Invalid key=value syntax");
+ let k = param.substr(0, idx);
+ let v = JSON.parse(param.substr(idx+1));
+ params[k] = v;
+ }
+ taskmaster.pushTask(args.taskname, params);
+ this._console = GSystem.Console.get();
+ loop.run();
+ if (!this._failed)
+ print("Success!")
+ },
+
+ _onTaskCompleted: function(taskmaster, task, success, error) {
+ if (success) {
+ print("Task " + task.name + " complete: " + task._workdir.get_path());
+ } else {
+ this._failed = true;
+ print("Task " + task.name + " failed: " + task._workdir.get_path());
+ }
+ },
+
+ _onTasksComplete: function(success, err) {
+ if (!success)
+ this._err = err;
+ this._loop.quit();
+ }
+});
diff --git a/src/ostbuild/js/builtins/run_task.js b/src/ostbuild/js/builtins/run_task.js
new file mode 100644
index 0000000..4a34c32
--- /dev/null
+++ b/src/ostbuild/js/builtins/run_task.js
@@ -0,0 +1,55 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const Task = imports.task;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const RunTask = new Lang.Class({
+ Name: 'RunTask',
+ Extends: Builtin.Builtin,
+
+ DESCRIPTION: "Internal helper to execute a task",
+
+ _init: function() {
+ this.parent();
+ this.parser.addArgument('taskName');
+ this.parser.addArgument('parameters');
+ },
+
+ execute: function(args, loop, cancellable) {
+ let taskset = Task.TaskSet.prototype.getInstance();
+ let [taskDef, vars] = taskset.getTask(args.taskName);
+ let params = JSON.parse(args.parameters);
+ let instance = new taskDef(null, args.taskName, vars, params);
+ instance.execute(cancellable);
+ }
+});
diff --git a/src/ostbuild/js/jsondb.js b/src/ostbuild/js/jsondb.js
index c61b311..d1d4b8d 100644
--- a/src/ostbuild/js/jsondb.js
+++ b/src/ostbuild/js/jsondb.js
@@ -99,6 +99,16 @@ const JsonDB = new Lang.Class({
return JsonUtil.loadJson(this._path.get_child(path.get_basename()), cancellable);
},
+ _updateIndex: function(cancellable) {
+ let files = this._getAll();
+ let fnames = [];
+ for (let i = 0; i < files.length; i++) {
+ fnames.push(files[i][3]);
+ }
+ let index = { files: fnames };
+ JsonUtil.writeJsonFileAtomic(this._path.get_child('index.json'), index, cancellable);
+ },
+
store: function(obj, cancellable) {
let files = this._getAll();
let latest = null;
@@ -133,6 +143,8 @@ const JsonDB = new Lang.Class({
}
}
+ this._updateIndex(cancellable);
+
return [targetPath, true];
}
});
diff --git a/src/ostbuild/js/jsonutil.js b/src/ostbuild/js/jsonutil.js
index 04a82ca..24deb88 100644
--- a/src/ostbuild/js/jsonutil.js
+++ b/src/ostbuild/js/jsonutil.js
@@ -27,6 +27,44 @@ function writeJsonToStream(stream, data, cancellable) {
stream.write_bytes(new GLib.Bytes(buf), cancellable);
}
+function writeJsonToStreamAsync(stream, data, cancellable, onComplete) {
+ let buf = JSON.stringify(data, null, " ");
+ stream.write_bytes_async(new GLib.Bytes(buf), GLib.PRIORITY_DEFAULT,
+ cancellable, function(stream, result) {
+ let err = null;
+ try {
+ stream.write_bytes_finish(result);
+ } catch (e) {
+ err = e;
+ }
+ onComplete(err != null, err);
+ });
+}
+
+function loadJsonFromStream(stream, cancellable) {
+ let membuf = Gio.MemoryOutputStream.new_resizable();
+ membuf.splice(stream, Gio.OutputStreamSpliceFlags.CLOSE_TARGET, cancellable);
+ let bytes = membuf.steal_as_bytes();
+ return JSON.parse(bytes.toArray().toString());
+}
+
+function loadJsonFromStreamAsync(stream, cancellable, onComplete) {
+ let membuf = Gio.MemoryOutputStream.new_resizable();
+ membuf.splice_async(stream, Gio.OutputStreamSpliceFlags.CLOSE_TARGET, GLib.PRIORITY_DEFAULT,
+ cancellable, function(stream, result) {
+ let err = null;
+ let res = null;
+ try {
+ stream.splice_finish(result);
+ let bytes = membuf.steal_as_bytes();
+ res = JSON.parse(bytes.toArray().toString());
+ } catch (e) {
+ err = e;
+ }
+ onComplete(res, err);
+ });
+}
+
function writeJsonFileAtomic(path, data, cancellable) {
let s = path.replace(null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable);
writeJsonToStream(s, data, cancellable);
diff --git a/src/ostbuild/js/jsutil.js b/src/ostbuild/js/jsutil.js
new file mode 100644
index 0000000..7b37a37
--- /dev/null
+++ b/src/ostbuild/js/jsutil.js
@@ -0,0 +1,23 @@
+// Copyright (C) 2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+function stringEndswith(s, suffix) {
+ let i = s.lastIndexOf(suffix);
+ if (i == -1)
+ return false;
+ return i == s.length - suffix.length;
+}
diff --git a/src/ostbuild/js/main.js b/src/ostbuild/js/main.js
index ae92800..0f1e2e2 100755
--- a/src/ostbuild/js/main.js
+++ b/src/ostbuild/js/main.js
@@ -26,7 +26,9 @@ const BUILTINS = ['autobuilder',
'resolve',
'build',
'build-disks',
+ 'make',
'shell',
+ 'run-task',
'qa-make-disk',
'qa-pull-deploy',
'qa-smoketest'];
diff --git a/src/ostbuild/js/procutil.js b/src/ostbuild/js/procutil.js
index 0af1ab3..3c30ab3 100644
--- a/src/ostbuild/js/procutil.js
+++ b/src/ostbuild/js/procutil.js
@@ -92,7 +92,7 @@ function _runSyncGetOutputInternal(argv, cancellable, params, splitLines) {
let [line, len] = dataIn.read_line_utf8(cancellable);
if (line == null)
break;
- result += line;
+ result += (line + '\n');
}
}
_wait_sync_check_internal(proc, cancellable);
diff --git a/src/ostbuild/js/snapshot.js b/src/ostbuild/js/snapshot.js
index 51dca5f..fbe760c 100644
--- a/src/ostbuild/js/snapshot.js
+++ b/src/ostbuild/js/snapshot.js
@@ -21,14 +21,17 @@ const Lang = imports.lang;
const JsonDB = imports.jsondb;
const JsonUtil = imports.jsonutil;
+const Vcs = imports.vcs;
const Params = imports.params;
function _componentDict(snapshot) {
let r = {};
let components = snapshot['components'];
- for (let i = 0; i< components.length; i++) {
+ for (let i = 0; i < components.length; i++) {
let component = components[i];
let name = component['name'];
+ if (r[name])
+ throw new Error("Duplicate component name " + name);
r[name] = component;
}
let patches = snapshot['patches'];
@@ -66,15 +69,69 @@ function snapshotDiff(a, b) {
const Snapshot = new Lang.Class({
Name: 'Snapshot',
- _init: function(data, path) {
+ _init: function(data, path, params) {
+ params = Params.parse(params, { prepareResolve: false });
this.data = data;
this.path = path;
+ if (params.prepareResolve) {
+ data['patches'] = this._resolveComponent(data, data['patches']);
+ data['base'] = this._resolveComponent(data, data['base']);
+ for (let i = 0; i < data['components'].length; i++) {
+ let component = this._resolveComponent(data, data['components'][i]);
+ data['components'][i] = component;
+ }
+ }
this._componentDict = _componentDict(data);
this._componentNames = [];
for (let k in this._componentDict)
this._componentNames.push(k);
},
+ _resolveComponent: function(manifest, componentMeta) {
+ let result = {};
+ Lang.copyProperties(componentMeta, result);
+ let origSrc = componentMeta['src'];
+
+ let didExpand = false;
+ let vcsConfig = manifest['vcsconfig'];
+ for (let vcsprefix in vcsConfig) {
+ let expansion = vcsConfig[vcsprefix];
+ let prefix = vcsprefix + ':';
+ if (origSrc.indexOf(prefix) == 0) {
+ result['src'] = expansion + origSrc.substr(prefix.length);
+ didExpand = true;
+ break;
+ }
+ }
+
+ let name = componentMeta['name'];
+ let src, idx, name;
+ if (name == undefined) {
+ if (didExpand) {
+ src = origSrc;
+ idx = src.lastIndexOf(':');
+ name = src.substr(idx+1);
+ } else {
+ src = result['src'];
+ idx = src.lastIndexOf('/');
+ name = src.substr(idx+1);
+ }
+ let i = name.lastIndexOf('.git');
+ if (i != -1 && i == name.length - 4) {
+ name = name.substr(0, name.length - 4);
+ }
+ name = name.replace(/\//g, '-');
+ result['name'] = name;
+ }
+
+ let branchOrTag = result['branch'] || result['tag'];
+ if (!branchOrTag) {
+ result['branch'] = 'master';
+ }
+
+ return result;
+ },
+
_expandComponent: function(component) {
let r = {};
Lang.copyProperties(component, r);
@@ -98,6 +155,10 @@ const Snapshot = new Lang.Class({
return this._componentNames;
},
+ getComponentMap: function() {
+ return this._componentDict;
+ },
+
getComponent: function(name, allowNone) {
let r = this._componentDict[name] || null;
if (!r && !allowNone)
@@ -118,5 +179,15 @@ const Snapshot = new Lang.Class({
getExpanded: function(name) {
return this._expandComponent(this.getComponent(name));
+ },
+
+ getVcsInfo: function(name) {
+ let component = this.getComponent(name);
+ let src = component['src']
+ let [keytype, uri] = Vcs.parseSrcKey(src);
+ let branch = component['branch'];
+ let tag = component['tag'];
+ let branchOrTag = branch || tag;
+ return [keytype, uri, branchOrTag];
}
});
diff --git a/src/ostbuild/js/task.js b/src/ostbuild/js/task.js
new file mode 100644
index 0000000..0f6c3a6
--- /dev/null
+++ b/src/ostbuild/js/task.js
@@ -0,0 +1,381 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const format = imports.format;
+const Lang = imports.lang;
+const Signals = imports.signals;
+
+const GSystem = imports.gi.GSystem;
+const Config = imports.config;
+const Params = imports.params;
+const JsonUtil = imports.jsonutil;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const BuildUtil = imports.buildutil;
+
+var _tasksetInstance = null;
+const TaskSet = new Lang.Class({
+ Name: 'TaskSet',
+
+ _init: function() {
+ this._tasks = [];
+ let taskdir = Gio.File.new_for_path(GLib.getenv('OSTBUILD_DATADIR')).resolve_relative_path('js/tasks');
+ let denum = taskdir.enumerate_children('standard::*', 0, null);
+ let finfo;
+
+ for (let taskmodname in imports.tasks) {
+ let taskMod = imports.tasks[taskmodname];
+ for (let defname in taskMod) {
+ if (defname.indexOf('Task') !== 0
+ || defname == 'Task')
+ continue;
+ let cls = taskMod[defname];
+ this.register(cls);
+ }
+ }
+ },
+
+ register: function(taskdef) {
+ this._tasks.push(taskdef);
+ },
+
+ getAllTasks: function() {
+ return this._tasks;
+ },
+
+ getTask: function(taskName, params) {
+ params = Params.parse(params, { allowNone: false })
+ for (let i = 0; i < this._tasks.length; i++) {
+ let taskDef = this._tasks[i];
+ let pattern = taskDef.prototype.TaskPattern;
+ let re = pattern[0];
+ let match = re.exec(taskName);
+ if (!match)
+ continue;
+ let vars = {};
+ for (let i = 1; i < pattern.length; i++) {
+ vars[pattern[i]] = match[i];
+ }
+ return [taskDef, vars];
+ }
+ if (!params.allowNone)
+ throw new Error("No task definition matches " + taskName);
+ return null;
+ },
+
+ getInstance: function() {
+ if (!_tasksetInstance)
+ _tasksetInstance = new TaskSet();
+ return _tasksetInstance;
+ }
+});
+
+const TaskMaster = new Lang.Class({
+ Name: 'TaskMaster',
+
+ _init: function(path, params) {
+ params = Params.parse(params, {onEmpty: null});
+ this.path = path;
+ this.maxConcurrent = GLib.get_num_processors();
+ this._onEmpty = params.onEmpty;
+ this.cancellable = null;
+ this._idleRecalculateId = 0;
+ this._executing = [];
+ this._pendingTasksList = [];
+ this._seenTasks = {};
+ this._taskErrors = {};
+ this._caughtError = false;
+
+ this._taskset = TaskSet.prototype.getInstance();
+ },
+
+ pushTask: function(taskName, parameters) {
+ if (this.isTaskQueued(taskName))
+ return;
+ let [taskDef, vars] = this._taskset.getTask(taskName);
+ let instance = new taskDef(this, taskName, vars, parameters);
+ instance.onComplete = Lang.bind(this, this._onComplete, instance);
+ this._pendingTasksList.push(instance);
+ this._queueRecalculate();
+ },
+
+ isTaskQueued: function(taskName) {
+ for (let i = 0; i < this._pendingTasksList.length; i++) {
+ let pending = this._pendingTasksList[i];
+ if (pending.TaskName == taskName)
+ return true;
+ }
+ for (let i = 0; i < this._executing.length; i++) {
+ let executingTask = this._executing[i];
+ if (executingTask.TaskName == taskName)
+ return true;
+ }
+ return false;
+ },
+
+ getTaskState: function() {
+ let r = [];
+ for (let i = 0; i < this._pendingTasksList.length; i++) {
+ r.push({running: false, task: this._pendingTasksList[i] });
+ }
+ for (let i = 0; i < this._executing.length; i++) {
+ r.push({running: true, task: this._executing[i] });
+ }
+ return r;
+ },
+
+ _queueRecalculate: function() {
+ if (this._idleRecalculateId > 0)
+ return;
+ this._idleRecalculateId = GLib.idle_add(GLib.PRIORITY_DEFAULT, Lang.bind(this, this._recalculate));
+ },
+
+ _recalculate: function() {
+ this._idleRecalculateId = 0;
+
+ if (this._executing.length == 0 &&
+ this._pendingTasksList.length == 0) {
+ this._onEmpty(true, null);
+ return;
+ } else if (this._pendingTasksList.length == 0) {
+ return;
+ }
+
+ this._reschedule();
+ },
+
+ _onComplete: function(success, error, task) {
+ this.emit('task-complete', task, success, error);
+ let idx = -1;
+ for (let i = 0; i < this._executing.length; i++) {
+ let executingTask = this._executing[i];
+ if (executingTask !== task)
+ continue;
+ idx = i;
+ break;
+ }
+ if (idx == -1)
+ throw new Error("TaskMaster: Internal error - Failed to find completed task:" + task.TaskName);
+ this._executing.splice(idx, 1);
+ this._queueRecalculate();
+ },
+
+ _reschedule: function() {
+ while (this._executing.length < this.maxConcurrent &&
+ this._pendingTasksList.length > 0) {
+ let task = this._pendingTasksList.shift();
+ task._executeInSubprocessInternal(this.cancellable);
+ this._executing.push(task);
+ }
+ }
+});
+Signals.addSignalMethods(TaskMaster.prototype);
+
+const TaskDef = new Lang.Class({
+ Name: 'TaskDef',
+
+ TaskPattern: null,
+
+ PreserveStdout: true,
+ RetainFailed: 1,
+ RetainSuccess: 5,
+
+ DefaultParameters: {},
+
+ _VERSION_RE: /^(\d+\d\d\d\d)\.(\d+)$/,
+
+ _init: function(taskmaster, name, vars, parameters) {
+ this.taskmaster = taskmaster;
+ this.name = name;
+ this.vars = vars;
+ this.parameters = Params.parse(parameters, this.DefaultParameters);
+
+ this.config = Config.get();
+ this.workdir = Gio.File.new_for_path(this.config.getGlobal('workdir'));
+ this.resultdir = this.workdir.get_child('results');
+ this.mirrordir = Gio.File.new_for_path(this.config.getGlobal('mirrordir'));
+ this.libdir = Gio.File.new_for_path(GLib.getenv('OSTBUILD_LIBDIR'));
+ this.repo = this.workdir.get_child('repo');
+ },
+
+ getDepends: function() {
+ return [];
+ },
+
+ _getResultDb: function(taskname) {
+ let path = this.resultdir.resolve_relative_path(taskname);
+ return new JsonDB.JsonDB(path);
+ },
+
+ _loadVersionsFrom: function(dir, cancellable) {
+ let e = dir.enumerate_children('standard::*', Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
+ let info;
+ let results = [];
+ while ((info = e.next_file(cancellable)) != null) {
+ let name = info.get_name();
+ let match = this._VERSION_RE.exec(name);
+ if (!match)
+ continue;
+ results.push(name);
+ }
+ results.sort(BuildUtil.compareVersions);
+ return results;
+ },
+
+ _cleanOldVersions: function(dir, retain, cancellable) {
+ let versions = this._loadVersionsFrom(dir, cancellable);
+ while (versions.length > retain) {
+ let child = dir.get_child(versions.shift());
+ GSystem.shutil_rm_rf(child, cancellable);
+ }
+ },
+
+ execute: function(cancellable) {
+ throw new Error("Not implemented");
+ },
+
+ _loadAllVersions: function(cancellable) {
+ let allVersions = [];
+
+ let successVersions = this._loadVersionsFrom(this._successDir, cancellable);
+ for (let i = 0; i < successVersions.length; i++) {
+ allVersions.push([true, successVersions[i]]);
+ }
+
+ let failedVersions = this._loadVersionsFrom(this._failedDir, cancellable);
+ for (let i = 0; i < failedVersions.length; i++) {
+ allVersions.push([false, failedVersions[i]]);
+ }
+
+ allVersions.sort(function (a, b) {
+ let [successA, versionA] = a;
+ let [successB, versionB] = b;
+ return BuildUtil.compareVersions(versionA, versionB);
+ });
+
+ return allVersions;
+ },
+
+ _executeInSubprocessInternal: function(cancellable) {
+ this._cancellable = cancellable;
+
+ this._startTimeMillis = GLib.get_monotonic_time() / 1000;
+
+ this.dir = this.taskmaster.path.resolve_relative_path(this.name);
+ GSystem.file_ensure_directory(this.dir, true, cancellable);
+
+ this._successDir = this.dir.get_child('successful');
+ GSystem.file_ensure_directory(this._successDir, true, cancellable);
+ this._failedDir = this.dir.get_child('failed');
+ GSystem.file_ensure_directory(this._failedDir, true, cancellable);
+
+ let allVersions = this._loadAllVersions(cancellable);
+
+ let currentTime = GLib.DateTime.new_now_utc();
+
+ let currentYmd = Format.vprintf('%d%02d%02d', [currentTime.get_year(),
+ currentTime.get_month(),
+ currentTime.get_day_of_month()]);
+ let version = null;
+ if (allVersions.length > 0) {
+ let [lastSuccess, lastVersion] = allVersions[allVersions.length-1];
+ let match = this._VERSION_RE.exec(lastVersion);
+ if (!match) throw new Error();
+ let lastYmd = match[1];
+ let lastSerial = match[2];
+ if (lastYmd == currentYmd) {
+ version = currentYmd + '.' + (parseInt(lastSerial) + 1);
+ }
+ }
+ if (version === null) {
+ version = currentYmd + '.0';
+ }
+
+ this._version = version;
+ this._workdir = this.dir.get_child(version);
+ GSystem.shutil_rm_rf(this._workdir, cancellable);
+ GSystem.file_ensure_directory(this._workdir, true, cancellable);
+
+ let baseArgv = ['ostbuild', 'run-task', this.name, JSON.stringify(this.parameters)];
+ let context = new GSystem.SubprocessContext({ argv: baseArgv });
+ context.set_cwd(this._workdir.get_path());
+ context.set_stdin_disposition(GSystem.SubprocessStreamDisposition.PIPE);
+ if (this.PreserveStdout) {
+ let outPath = this._workdir.get_child('output.txt');
+ context.set_stdout_file_path(outPath.get_path());
+ context.set_stderr_disposition(GSystem.SubprocessStreamDisposition.STDERR_MERGE);
+ } else {
+ context.set_stdout_disposition(GSystem.SubprocessStreamDisposition.NULL);
+ let errPath = this._workdir.get_child('errors.txt');
+ context.set_stderr_file_path(errPath.get_path());
+ }
+ this._proc = new GSystem.Subprocess({ context: context });
+ this._proc.init(cancellable);
+
+ this._proc.wait(cancellable, Lang.bind(this, this._onChildExited));
+ },
+
+ _updateIndex: function(cancellable) {
+ let allVersions = this._loadAllVersions(cancellable);
+
+ let fileList = [];
+ for (let i = 0; i < allVersions.length; i++) {
+ let [successful, version] = allVersions[i];
+ let fname = (successful ? 'successful/' : 'failed/') + version;
+ fileList.push(fname);
+ }
+
+ let index = { files: fileList };
+ JsonUtil.writeJsonFileAtomic(this.dir.get_child('index.json'), index, cancellable);
+ },
+
+ _onChildExited: function(proc, result) {
+ let cancellable = this._cancellable;
+ let [success, errmsg] = ProcUtil.asyncWaitCheckFinish(proc, result);
+ let target;
+
+ let elapsedMillis = GLib.get_monotonic_time() / 1000 - this._startTimeMillis;
+ let meta = { taskMetaVersion: 0,
+ taskVersion: this._version,
+ success: success,
+ errmsg: errmsg,
+ elapsedMillis: elapsedMillis };
+ JsonUtil.writeJsonFileAtomic(this._workdir.get_child('meta.json'), meta, cancellable);
+
+ if (!success) {
+ target = this._failedDir.get_child(this._version);
+ GSystem.file_rename(this._workdir, target, null);
+ this._workdir = target;
+ this._cleanOldVersions(this._failedDir, this.RetainFailed, null);
+ this.onComplete(success, errmsg);
+ } else {
+ target = this._successDir.get_child(this._version);
+ GSystem.file_rename(this._workdir, target, null);
+ this._workdir = target;
+ this._cleanOldVersions(this._successDir, this.RetainSuccess, null);
+ this.onComplete(success, null);
+ }
+ // Also remove any old interrupted versions
+ this._cleanOldVersions(this.dir, 0, null);
+
+ this._updateIndex(cancellable);
+
+ BuildUtil.atomicSymlinkSwap(this.dir.get_child('current'), target, cancellable);
+ }
+});
diff --git a/src/ostbuild/js/tasks/task-bdiff.js b/src/ostbuild/js/tasks/task-bdiff.js
new file mode 100644
index 0000000..1d249f6
--- /dev/null
+++ b/src/ostbuild/js/tasks/task-bdiff.js
@@ -0,0 +1,155 @@
+// Copyright (C) 2011,2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const Task = imports.task;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const StreamUtil = imports.streamutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const TaskBdiff = new Lang.Class({
+ Name: "TaskBdiff",
+ Extends: Task.TaskDef,
+
+ TaskPattern: [/bdiff\/(.*?)$/, 'prefix'],
+
+ TaskAfterPrefix: '/build/',
+
+ _gitLogToJson: function(repoDir, specification) {
+ let log = ProcUtil.runSyncGetOutputLines(['git', 'log', '--format=email', specification],
+ null,
+ { cwd: repoDir });
+ let r = [];
+ if (log.length == 0)
+ return r;
+ let currentItem = null;
+ let parsingHeaders = false;
+ let fromRegex = /^From ([0-9a-f]{40}) /;
+ for (let i = 0; i < log.length; i++) {
+ let line = log[i];
+ let match = fromRegex.exec(line);
+ if (match) {
+ if (currentItem !== null) {
+ r.push(currentItem);
+ }
+ currentItem = {'Checksum': match[1]};
+ parsingHeaders = true;
+ } else if (parsingHeaders) {
+ if (line.length == 0) {
+ parsingHeaders = false;
+ } else {
+ let idx = line.indexOf(':');
+ let k = line.substr(0, idx);
+ let v = line.substr(idx+1);
+ currentItem[k] = v;
+ }
+ }
+ }
+ return r;
+ },
+
+ _diffstat: function(repoDir, specification) {
+ return ProcUtil.runSyncGetOutputUTF8(['git', 'diff', '--stat', specification], null,
+ { cwd: repoDir });
+ },
+
+ execute: function(cancellable) {
+ let prefix = this.vars['prefix'];
+
+ this.subworkdir = Gio.File.new_for_path('.');
+
+ let builddb = this._getResultDb('build/' + prefix);
+ let latestPath = builddb.getLatestPath();
+ if (!latestPath)
+ throw new Error("No builds!")
+ let latestBuildVersion = builddb.parseVersionStr(latestPath.get_basename());
+
+ let previousPath = builddb.getPreviousPath(latestPath);
+ if (!previousPath)
+ throw new Error("No build previous to " + latestBuildVersion)
+
+ let latestBuildData = builddb.loadFromPath(latestPath, cancellable);
+ let latestBuildSnapshot = new Snapshot.Snapshot(latestBuildData['snapshot'], null);
+ let previousBuildData = builddb.loadFromPath(previousPath, cancellable);
+ let previousBuildSnapshot = new Snapshot.Snapshot(previousBuildData['snapshot'], null);
+
+ let added = [];
+ let modified = [];
+ let removed = [];
+
+ let result = {fromBuildVersion: builddb.parseVersionStr(previousPath.get_basename()),
+ toBuildVersion: builddb.parseVersionStr(latestPath.get_basename()),
+ fromSrcVersion: builddb.parseVersionStr(previousBuildData['snapshotName']),
+ toSrcVersion: builddb.parseVersionStr(latestBuildData['snapshotName']),
+ added: added,
+ modified: modified,
+ removed: removed};
+
+ let modifiedNames = [];
+
+ let latestComponentMap = latestBuildSnapshot.getComponentMap();
+ let previousComponentMap = previousBuildSnapshot.getComponentMap();
+ for (let componentName in latestComponentMap) {
+ let componentA = latestBuildSnapshot.getComponent(componentName);
+ let componentB = previousBuildSnapshot.getComponent(componentName, true);
+
+ if (componentB === null)
+ added.push(componentName);
+ else if (componentB.revision != componentA.revision)
+ modifiedNames.push(componentName);
+ }
+ for (let componentName in previousComponentMap) {
+ let componentA = latestBuildSnapshot.getComponent(componentName, true);
+
+ if (componentA === null)
+ removed.push(componentName);
+ }
+
+ for (let i = 0; i < modifiedNames.length; i++) {
+ let componentName = modifiedNames[i];
+ let latestComponent = latestBuildSnapshot.getComponent(componentName);
+ let previousComponent = previousBuildSnapshot.getComponent(componentName);
+ let latestRevision = latestComponent.revision;
+ let previousRevision = previousComponent.revision;
+ let [keytype, uri, branchOrTag] = latestBuildSnapshot.getVcsInfo(componentName);
+ let mirrordir = Vcs.getMirrordir(this.mirrordir, keytype, uri);
+
+ let gitlog = this._gitLogToJson(mirrordir, previousRevision + '...' + latestRevision);
+ let diffstat = this._diffstat(mirrordir, previousRevision + '..' + latestRevision);
+ modified.push({ previous: previousComponent,
+ latest: latestComponent,
+ gitlog: gitlog,
+ diffstat: diffstat });
+ }
+
+ let bdiffdb = this._getResultDb('bdiff/' + prefix);
+ bdiffdb.store(result, cancellable);
+ }
+});
diff --git a/src/ostbuild/js/builtins/build.js b/src/ostbuild/js/tasks/task-build.js
similarity index 92%
rename from src/ostbuild/js/builtins/build.js
rename to src/ostbuild/js/tasks/task-build.js
index 7c54a8a..24f4b80 100644
--- a/src/ostbuild/js/builtins/build.js
+++ b/src/ostbuild/js/tasks/task-build.js
@@ -23,7 +23,7 @@ const Format = imports.format;
const GSystem = imports.gi.GSystem;
const Builtin = imports.builtin;
-const SubTask = imports.subtask;
+const Task = imports.task;
const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const StreamUtil = imports.streamutil;
@@ -35,13 +35,15 @@ const Vcs = imports.vcs;
const ArgParse = imports.argparse;
const OPT_COMMON_CFLAGS = {'i686': '-O2 -g -m32 -march=i686 -mtune=atom -fasynchronous-unwind-tables',
- 'x86_64': '-O2 -g -m64 -mtune=generic'}
+ 'x86_64': '-O2 -g -m64 -mtune=generic'};
-const Build = new Lang.Class({
- Name: "Build",
- Extends: Builtin.Builtin,
+const TaskBuild = new Lang.Class({
+ Name: "TaskBuild",
+ Extends: Task.TaskDef,
- DESCRIPTION: "Build multiple components and generate trees",
+ TaskPattern: [/build\/(.*?)$/, 'prefix'],
+
+ TaskAfterPrefix: '/resolve/',
_resolveRefs: function(refs) {
if (refs.length == 0)
@@ -290,6 +292,7 @@ const Build = new Lang.Class({
let prefix = this._snapshot.data['prefix'];
let buildname = Format.vprintf('%s/%s/%s', [prefix, basename, architecture]);
+ let unixBuildname = buildname.replace(/\//g, '_');
let buildRef = 'components/' + buildname;
let currentVcsVersion = component['revision'];
@@ -320,16 +323,13 @@ const Build = new Lang.Class({
let patchdir;
if (expandedComponent['patches']) {
let patchesRevision = expandedComponent['patches']['revision'];
- if (this.args.patches_path) {
- patchdir = Gio.File.new_for_path(this.args.patches_path);
- } else if (this._cachedPatchdirRevision == patchesRevision) {
+ if (this._cachedPatchdirRevision == patchesRevision) {
patchdir = this.patchdir;
} else {
patchdir = Vcs.checkoutPatches(this.mirrordir,
this.patchdir,
expandedComponent,
- cancellable,
- {patchesPath: this.args.patches_path});
+ cancellable);
this._cachedPatchdirRevision = patchesRevision;
}
if ((previousMetadata != null) &&
@@ -366,33 +366,31 @@ const Build = new Lang.Class({
}
}
- let taskdir = this.workdir.get_child('tasks');
- let buildTaskset = new SubTask.TaskSet(taskdir.get_child(buildname));
-
- let workdir = buildTaskset.prepare();
+ let cwd = Gio.File.new_for_path('.');
+ let buildWorkdir = cwd.get_child('tmp-' + unixBuildname);
+ GSystem.file_ensure_directory(buildWorkdir, true, cancellable);
- let tempMetadataPath = workdir.get_child('_ostbuild-meta.json');
+ let tempMetadataPath = buildWorkdir.get_child('_ostbuild-meta.json');
JsonUtil.writeJsonFileAtomic(tempMetadataPath, expandedComponent, cancellable);
- let componentSrc = workdir.get_child(basename);
+ let componentSrc = buildWorkdir.get_child(basename);
let childArgs = ['ostbuild', 'checkout', '--snapshot=' + this._snapshot.path.get_path(),
'--checkoutdir=' + componentSrc.get_path(),
'--metadata-path=' + tempMetadataPath.get_path(),
'--overwrite', basename];
- if (this.args.patches_path)
- childArgs.push('--patches-path=' + this.args.patches_path);
- else if (patchdir)
+ if (patchdir) {
childArgs.push('--patches-path=' + patchdir.get_path());
- ProcUtil.runSync(childArgs, cancellable, { logInitiation: true });
+ }
+ ProcUtil.runSync(childArgs, cancellable);
GSystem.file_unlink(tempMetadataPath, cancellable);
- let componentResultdir = workdir.get_child('results');
+ let componentResultdir = buildWorkdir.get_child('results');
GSystem.file_ensure_directory(componentResultdir, true, cancellable);
- let rootdir = this._composeBuildroot(workdir, basename, architecture, cancellable);
+ let rootdir = this._composeBuildroot(buildWorkdir, basename, architecture, cancellable);
- let tmpdir=workdir.get_child('tmp');
+ let tmpdir=buildWorkdir.get_child('tmp');
GSystem.file_ensure_directory(tmpdir, true, cancellable);
let srcCompileOnePath = this.libdir.get_child('ostree-build-compile-one');
@@ -424,26 +422,11 @@ const Build = new Lang.Class({
let context = new GSystem.SubprocessContext({ argv: childArgs });
context.set_environment(ProcUtil.objectToEnvironment(envCopy));
-
- let mainContext = new GLib.MainContext();
- mainContext.push_thread_default();
- let loop = GLib.MainLoop.new(mainContext, true);
- let t;
- try {
- t = buildTaskset.start(context, cancellable, Lang.bind(this, this._onBuildComplete, loop));
- print("Started child process " + context.argv.map(GLib.shell_quote).join(' '));
- loop.run();
- } finally {
- mainContext.pop_thread_default();
- }
- let buildSuccess = this._currentBuildSucceded;
- let msg = this._currentBuildSuccessMsg;
-
- if (!buildSuccess) {
- this._analyzeBuildFailure(t, architecture, component, componentSrc,
- currentVcsVersion, previousVcsVersion, cancellable);
- throw new Error("Build failure in component " + buildname + " : " + msg);
- }
+
+ let proc = new GSystem.Subprocess({ context: context });
+ proc.init(cancellable);
+ print("Started child process " + context.argv.map(GLib.shell_quote).join(' '));
+ proc.wait_sync_check(cancellable);
let recordedMetaPath = componentResultdir.get_child('_ostbuild-meta.json');
JsonUtil.writeJsonFileAtomic(recordedMetaPath, expandedComponent, cancellable);
@@ -472,7 +455,7 @@ const Build = new Lang.Class({
if (statoverridePath != null)
GSystem.file_unlink(statoverridePath, cancellable);
- GSystem.shutil_rm_rf(tmpdir, cancellable);
+ GSystem.shutil_rm_rf(buildWorkdir, cancellable);
let ostreeRevision = this._saveComponentBuild(buildname, expandedComponent, cancellable);
@@ -485,8 +468,7 @@ const Build = new Lang.Class({
let runtimeName = 'bases/' + base['runtime'];
let develName = 'bases/' + base['devel'];
- let rootdir = this.workdir.get_child('roots');
- let composeRootdir = rootdir.get_child(target['name']);
+ let composeRootdir = this.subworkdir.get_child(target['name']);
GSystem.shutil_rm_rf(composeRootdir, cancellable);
GSystem.file_ensure_directory(composeRootdir, true, cancellable);
@@ -623,8 +605,7 @@ const Build = new Lang.Class({
Lang.copyProperties(BuildUtil.BUILD_ENV, env);
env['DL_DIR'] = downloads.get_path();
env['SSTATE_DIR'] = sstateDir.get_path();
- ProcUtil.runSync(cmd, cancellable, { logInitiation: true,
- env:ProcUtil.objectToEnvironment(env) });
+ ProcUtil.runSync(cmd, cancellable, {env:ProcUtil.objectToEnvironment(env)});
let componentTypes = ['runtime', 'devel'];
for (let i = 0; i < componentTypes.length; i++) {
@@ -642,26 +623,30 @@ const Build = new Lang.Class({
builtRevisionPath.replace_contents(basemeta['revision'], null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable);
},
- _init: function() {
- this.parent();
- this.parser.addArgument('--prefix');
- this.parser.addArgument('--snapshot');
- this.parser.addArgument('--patches-path');
+ execute: function(cancellable) {
+ let prefix = this.vars['prefix'];
+
+ this.subworkdir = Gio.File.new_for_path('.');
this.forceBuildComponents = {};
this.cachedPatchdirRevision = null;
- },
-
- execute: function(args, loop, cancellable) {
- this._initSnapshot(args.prefix, args.snapshot, cancellable);
- this.args = args;
+
+ this.prefix = prefix;
+ this.patchdir = this.workdir.get_child('patches');
+ let snapshotDir = this.workdir.get_child('snapshots');
+ let srcdb = new JsonDB.JsonDB(snapshotDir.get_child(prefix));
+ let snapshotPath = srcdb.getLatestPath();
+ let workingSnapshotPath = this.subworkdir.get_child(snapshotPath.get_basename());
+ GSystem.file_linkcopy(snapshotPath, workingSnapshotPath, Gio.FileCopyFlags.OVERWRITE,
+ cancellable);
+ let data = srcdb.loadFromPath(workingSnapshotPath, cancellable);
+ this._snapshot = new Snapshot.Snapshot(data, workingSnapshotPath);
let components = this._snapshot.data['components'];
- let buildresultDir = this.workdir.get_child('builds').get_child(this.prefix);
- let builddb = new JsonDB.JsonDB(buildresultDir);
+ let builddb = this._getResultDb('build/' + this.prefix);
- let targetSourceVersion = builddb.parseVersionStr(this._snapshot.path.get_basename())
+ let targetSourceVersion = builddb.parseVersionStr(this._snapshot.path.get_basename());
let haveLocalComponent = false;
for (let i = 0; i < components.length; i++) {
diff --git a/src/ostbuild/js/tasks/task-builddisks.js b/src/ostbuild/js/tasks/task-builddisks.js
new file mode 100644
index 0000000..c1bdf11
--- /dev/null
+++ b/src/ostbuild/js/tasks/task-builddisks.js
@@ -0,0 +1,141 @@
+// -*- indent-tabs-mode: nil; tab-width: 2; -*-
+// Copyright (C) 2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const ArgParse = imports.argparse;
+const Task = imports.task;
+const ProcUtil = imports.procutil;
+const BuildUtil = imports.buildutil;
+const LibQA = imports.libqa;
+const JsonDB = imports.jsondb;
+const Config = imports.config;
+const JsonUtil = imports.jsonutil;
+const GuestFish = imports.guestfish;
+
+const IMAGE_RETAIN_COUNT = 2;
+
+const TaskBuildDisks = new Lang.Class({
+ Name: 'TaskBuildDisks',
+ Extends: Task.TaskDef,
+
+ TaskPattern: [/builddisks\/(.*?)$/, 'prefix'],
+
+ TaskAfterPrefix: '/build/',
+
+ // Legacy
+ _VERSION_RE: /^(\d+)\.(\d+)$/,
+
+ execute: function(cancellable) {
+ let prefix = this.vars['prefix'];
+
+ let subworkdir = Gio.File.new_for_path('.');
+
+ let baseImageDir = this.workdir.get_child('images').get_child(prefix);
+ GSystem.file_ensure_directory(baseImageDir, true, cancellable);
+ let currentImageLink = baseImageDir.get_child('current');
+ let previousImageLink = baseImageDir.get_child('previous');
+
+ let builddb = this._getResultDb('build/' + prefix);
+
+ let latestPath = builddb.getLatestPath();
+ let buildVersion = builddb.parseVersionStr(latestPath.get_basename());
+ let buildData = builddb.loadFromPath(latestPath, cancellable);
+
+ let targetImageDir = baseImageDir.get_child(buildVersion);
+
+ if (targetImageDir.query_exists(null)) {
+ print("Already created " + targetImageDir.get_path());
+ return;
+ }
+
+ let workImageDir = subworkdir.get_child('images');
+ GSystem.file_ensure_directory(workImageDir, true, cancellable);
+
+ let targets = buildData['targets'];
+
+ let osname = buildData['snapshot']['osname'];
+
+ for (let targetName in targets) {
+ let targetRevision = buildData['targets'][targetName];
+ let squashedName = targetName.replace(/\//g, '_');
+ let diskName = prefix + '-' + squashedName + '-disk.qcow2';
+ let diskPath = workImageDir.get_child(diskName);
+ let prevPath = currentImageLink.get_child(diskName);
+ GSystem.shutil_rm_rf(diskPath, cancellable);
+ if (prevPath.query_exists(null)) {
+ LibQA.copyDisk(prevPath, diskPath, cancellable);
+ } else {
+ LibQA.createDisk(diskPath, cancellable);
+ }
+ ProcUtil.runSync(['ostbuild', 'qa-pull-deploy', diskPath.get_path(),
+ this.repo.get_path(), osname, targetName, targetRevision],
+ cancellable, { logInitiation: true });
+ }
+
+ GSystem.file_rename(workImageDir, targetImageDir, cancellable);
+
+ let currentInfo = null;
+ try {
+ currentInfo = currentImageLink.query_info('standard::symlink-target', Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
+ } catch (e) {
+ if (!e.matches(Gio.IOErrorEnum, Gio.IOErrorEnum.NOT_FOUND))
+ throw e;
+ }
+ if (currentInfo != null) {
+ let newPreviousTmppath = baseImageDir.get_child('previous-new.tmp');
+ let currentLinkTarget = currentInfo.get_symlink_target();
+ GSystem.shutil_rm_rf(newPreviousTmppath, cancellable);
+ newPreviousTmppath.make_symbolic_link(currentLinkTarget, cancellable);
+ GSystem.file_rename(newPreviousTmppath, previousImageLink, cancellable);
+ }
+ BuildUtil.atomicSymlinkSwap(baseImageDir.get_child('current'), targetImageDir, cancellable);
+
+ this._cleanOldVersions(baseImageDir, IMAGE_RETAIN_COUNT, cancellable);
+ },
+
+ _loadVersionsFrom: function(dir, cancellable) {
+ let e = dir.enumerate_children('standard::*', Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
+ let info;
+ let results = [];
+ while ((info = e.next_file(cancellable)) != null) {
+ let name = info.get_name();
+ let match = this._VERSION_RE.exec(name);
+ if (!match)
+ continue;
+ results.push(name);
+ }
+ results.sort(BuildUtil.compareVersions);
+ return results;
+ },
+
+ _cleanOldVersions: function(dir, retain, cancellable) {
+ let versions = this._loadVersionsFrom(dir, cancellable);
+ while (versions.length > retain) {
+ let child = dir.get_child(versions.shift());
+ GSystem.shutil_rm_rf(child, cancellable);
+ }
+ },
+
+});
diff --git a/src/ostbuild/js/tasks/task-resolve.js b/src/ostbuild/js/tasks/task-resolve.js
new file mode 100644
index 0000000..1080b8a
--- /dev/null
+++ b/src/ostbuild/js/tasks/task-resolve.js
@@ -0,0 +1,84 @@
+// Copyright (C) 2011 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const JsonDB = imports.jsondb;
+const Builtin = imports.builtin;
+const Task = imports.task;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const TaskResolve = new Lang.Class({
+ Name: "TaskResolve",
+ Extends: Task.TaskDef,
+
+ TaskPattern: [/resolve\/(.*?)$/, 'prefix'],
+
+ DefaultParameters: {fetchAll: false,
+ fetchComponents: [],
+ timeoutSec: 10},
+
+ execute: function(cancellable) {
+ this.prefix = this.vars['prefix'];
+ let manifest = this.config.getGlobal('manifest');
+ let manifestPath = Gio.File.new_for_path(manifest);
+ let data = JsonUtil.loadJson(manifestPath, cancellable);
+ this._snapshot = new Snapshot.Snapshot(data, manifestPath, { prepareResolve: true });
+
+ let gitMirrorArgs = ['ostbuild', 'git-mirror', '--timeout-sec=' + this.parameters.timeoutSec,
+ '--manifest=' + manifest];
+ if (this.parameters.fetchAll || this.parameters.fetchComponents.length > 0) {
+ gitMirrorArgs.push('--fetch');
+ gitMirrorArgs.push('-k');
+ gitMirrorArgs.push.apply(gitMirrorArgs, this.parameters.fetchComponents);
+ }
+ ProcUtil.runSync(gitMirrorArgs, cancellable, { logInitiation: true });
+
+ let componentNames = this._snapshot.getAllComponentNames();
+ for (let i = 0; i < componentNames.length; i++) {
+ let component = this._snapshot.getComponent(componentNames[i]);
+ let src = component['src'];
+ let [keytype, uri] = Vcs.parseSrcKey(src);
+ let branch = component['branch'];
+ let tag = component['tag'];
+ let branchOrTag = branch || tag;
+ let mirrordir = Vcs.ensureVcsMirror(this.mirrordir, keytype, uri, branchOrTag, cancellable);
+ let revision = Vcs.describeVersion(mirrordir, branchOrTag);
+ component['revision'] = revision;
+ }
+
+ let snapshotdir = this.workdir.get_child('snapshots');
+ this._src_db = new JsonDB.JsonDB(snapshotdir.get_child(this.prefix));
+ let [path, modified] = this._src_db.store(this._snapshot.data, cancellable);
+ if (modified) {
+ print("New source snapshot: " + path.get_path());
+ } else {
+ print("Source snapshot unchanged: " + path.get_path());
+ }
+ }
+});
diff --git a/src/ostbuild/js/builtins/qa_smoketest.js b/src/ostbuild/js/tasks/task-smoketest.js
similarity index 66%
rename from src/ostbuild/js/builtins/qa_smoketest.js
rename to src/ostbuild/js/tasks/task-smoketest.js
index 1708e3c..3722ad5 100644
--- a/src/ostbuild/js/builtins/qa_smoketest.js
+++ b/src/ostbuild/js/tasks/task-smoketest.js
@@ -26,31 +26,35 @@ const GSystem = imports.gi.GSystem;
const Builtin = imports.builtin;
const ArgParse = imports.argparse;
const ProcUtil = imports.procutil;
+const Task = imports.task;
const LibQA = imports.libqa;
+const JSUtil = imports.jsutil;
const TIMEOUT_SECONDS = 2 * 60;
-const QaSmoketest = new Lang.Class({
- Name: 'QaSmoketest',
- Extends: Builtin.Builtin,
- DESCRIPTION: "Test booting and logging in",
+const RequiredMessageIDs = ["39f53479d3a045ac8e11786248231fbf", // graphical.target
+ "f77379a8490b408bbe5f6940505a777b", // systemd-journald
+ "0ce153587afa4095832d233c17a88001" // gnome-session startup ok
+ ];
+const FailedMessageIDs = ["fc2e22bc6ee647b6b90729ab34a250b1", // coredump
+ "10dd2dc188b54a5e98970f56499d1f73" // gnome-session required component failed
+ ];
- RequiredMessageIDs: ["39f53479d3a045ac8e11786248231fbf", // graphical.target
- "f77379a8490b408bbe5f6940505a777b", // systemd-journald
- "0ce153587afa4095832d233c17a88001" // gnome-session startup ok
- ],
- FailedMessageIDs: ["fc2e22bc6ee647b6b90729ab34a250b1", // coredump
- "10dd2dc188b54a5e98970f56499d1f73" // gnome-session required component failed
- ],
+const SmoketestOne = new Lang.Class({
+ Name: 'SmoketestOne',
+ _fail: function(message) {
+ this._failed = true;
+ this._failedMessage = message;
+ },
+
_onQemuExited: function(proc, result) {
let [success, status] = ProcUtil.asyncWaitCheckFinish(proc, result);
this._qemu = null;
this._loop.quit();
if (!success) {
- this._failed = true;
- print("Qemu exited with status " + status);
+ this._fail("Qemu exited with status " + status);
}
},
@@ -59,7 +63,7 @@ const QaSmoketest = new Lang.Class({
for (let msgid in this._pendingRequiredMessageIds) {
print("Did not see MESSAGE_ID=" + msgid);
}
- this._failed = true;
+ this._fail("Timed out");
this._loop.quit();
},
@@ -72,8 +76,7 @@ const QaSmoketest = new Lang.Class({
this._journalDataStream.read_line_async(GLib.PRIORITY_DEFAULT, this._cancellable,
Lang.bind(this, this._onJournalReadLine));
} catch (e) {
- print("Open failed: " + e);
- this._failed = true;
+ this._fail("Journal open failed: " + e);
this._loop.quit();
}
},
@@ -84,10 +87,12 @@ const QaSmoketest = new Lang.Class({
try {
[line, len] = stream.read_line_finish_utf8(result);
} catch (e) {
- this._failed = true;
+ this._fail(e.toString());
this._loop.quit();
throw e;
}
+ if (this._done || this._failed)
+ return;
if (line) {
let data = JSON.parse(line);
let messageId = data['MESSAGE_ID'];
@@ -99,10 +104,9 @@ const QaSmoketest = new Lang.Class({
this._countPendingRequiredMessageIds--;
matched = true;
} else {
- for (let i = 0; i < this.FailedMessageIDs.length; i++) {
- if (messageId == this.FailedMessageIDs[i]) {
- print("Found failure message ID " + messageId);
- this._failed = true;
+ for (let i = 0; i < FailedMessageIDs.length; i++) {
+ if (messageId == FailedMessageIDs[i]) {
+ this._fail("Found failure message ID " + messageId);
this._loop.quit();
matched = true;
break;
@@ -116,12 +120,15 @@ const QaSmoketest = new Lang.Class({
Lang.bind(this, this._onJournalReadLine));
} else {
print("Found all required message IDs, exiting");
+ this._done = true;
this._loop.quit();
}
}
},
_onJournalChanged: function(monitor, file, otherFile, eventType) {
+ if (this._done || this._failed)
+ return;
if (!this._openedJournal) {
this._openedJournal = true;
file.read_async(GLib.PRIORITY_DEFAULT,
@@ -134,14 +141,10 @@ const QaSmoketest = new Lang.Class({
}
},
- _init: function() {
- this.parent();
- this.parser.addArgument('--monitor', { action: 'storeTrue' });
- this.parser.addArgument('diskpath');
- },
-
- execute: function(args, loop, cancellable) {
- this._loop = loop;
+ execute: function(subworkdir, prefix, diskPath, cancellable) {
+ print("Smoke testing disk " + diskPath.get_path());
+ this._loop = GLib.MainLoop.new(null, true);
+ this._done = false;
this._failed = false;
this._journalStream = null;
this._journalDataStream = null;
@@ -149,22 +152,19 @@ const QaSmoketest = new Lang.Class({
this._readingJournal = false;
this._pendingRequiredMessageIds = {};
this._countPendingRequiredMessageIds = 0;
- for (let i = 0; i < this.RequiredMessageIDs.length; i++) {
- this._pendingRequiredMessageIds[this.RequiredMessageIDs[i]] = true;
+ for (let i = 0; i < RequiredMessageIDs.length; i++) {
+ this._pendingRequiredMessageIds[RequiredMessageIDs[i]] = true;
this._countPendingRequiredMessageIds += 1;
}
this._cancellable = cancellable;
- let srcDiskpath = Gio.File.new_for_path(args.diskpath);
- let workdir = Gio.File.new_for_path('.');
-
let qemuArgs = [LibQA.getQemuPath()];
qemuArgs.push.apply(qemuArgs, LibQA.DEFAULT_QEMU_OPTS);
- let diskClone = workdir.get_child('qa-smoketest.qcow2');
+ let diskClone = subworkdir.get_child('smoketest-' + diskPath.get_basename());
GSystem.shutil_rm_rf(diskClone, cancellable);
- LibQA.createDiskSnapshot(srcDiskpath, diskClone, cancellable);
+ LibQA.createDiskSnapshot(diskPath, diskClone, cancellable);
let [gfmnt, mntdir] = LibQA.newReadWriteMount(diskClone, cancellable);
try {
LibQA.modifyBootloaderAppendKernelArgs(mntdir, ["console=ttyS0"], cancellable);
@@ -178,25 +178,17 @@ const QaSmoketest = new Lang.Class({
gfmnt.umount(cancellable);
}
- let consoleOutput = Gio.File.new_for_path('console.out');
- GSystem.shutil_rm_rf(consoleOutput, cancellable);
- let journalOutput = Gio.File.new_for_path('journal-json.txt');
- GSystem.shutil_rm_rf(journalOutput, cancellable);
+ let consoleOutput = subworkdir.get_child('console.out');
+ let journalOutput = subworkdir.get_child('journal-json.txt');
qemuArgs.push.apply(qemuArgs, ['-drive', 'file=' + diskClone.get_path() + ',if=virtio',
'-vnc', 'none',
- '-watchdog', 'ib700',
- '-watchdog-action', 'poweroff',
'-serial', 'file:' + consoleOutput.get_path(),
'-device', 'virtio-serial',
'-chardev', 'file,id=journaljson,path=' + journalOutput.get_path(),
'-device', 'virtserialport,chardev=journaljson,name=org.gnome.journaljson']);
- if (args.monitor)
- qemuArgs.push.apply(qemuArgs, ['-monitor', 'stdio']);
let qemuContext = new GSystem.SubprocessContext({ argv: qemuArgs });
- if (args.monitor)
- qemuContext.set_stdin_disposition(GSystem.SubprocessStreamDisposition.INHERIT);
let qemu = new GSystem.Subprocess({context: qemuContext});
this._qemu = qemu;
print("starting qemu");
@@ -207,19 +199,47 @@ const QaSmoketest = new Lang.Class({
let journalMonitor = journalOutput.monitor_file(0, cancellable);
journalMonitor.connect('changed', Lang.bind(this, this._onJournalChanged));
- GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT, TIMEOUT_SECONDS,
- Lang.bind(this, this._onTimeout));
+ let timeoutId = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT, TIMEOUT_SECONDS,
+ Lang.bind(this, this._onTimeout));
- loop.run();
+ this._loop.run();
if (this._qemu)
this._qemu.force_exit();
+
+ GLib.source_remove(timeoutId);
if (this._failed) {
- print("Exiting abnormally");
- return 1;
+ throw new Error(this._failedMessage);
+ }
+ print("Completed smoke testing of " + diskPath.get_basename());
+ }
+});
+
+const TaskSmoketest = new Lang.Class({
+ Name: 'TaskSmoketest',
+ Extends: Task.TaskDef,
+
+ TaskPattern: [/smoketest\/(.*?)$/, 'prefix'],
+
+ execute: function(cancellable) {
+ let prefix = this.vars['prefix'];
+
+ let imageDir = this.workdir.get_child('images').get_child(prefix);
+ let currentImages = imageDir.get_child('current');
+
+ let e = currentImages.enumerate_children('standard::*', Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS,
+ cancellable);
+ let info;
+ while ((info = e.next_file(cancellable)) != null) {
+ let name = info.get_name();
+ if (!JSUtil.stringEndswith(name, '.qcow2'))
+ continue;
+ let workdirName = 'work-' + name.replace(/\.qcow2$/, '');
+ let subworkdir = Gio.File.new_for_path(workdirName);
+ GSystem.file_ensure_directory(subworkdir, true, cancellable);
+ let smokeTest = new SmoketestOne();
+ smokeTest.execute(subworkdir, prefix, currentImages.get_child(name), cancellable);
}
- print("Complete!");
- return 0;
}
});
diff --git a/src/ostbuild/js/vcs.js b/src/ostbuild/js/vcs.js
index e2f82ad..3d324c2 100644
--- a/src/ostbuild/js/vcs.js
+++ b/src/ostbuild/js/vcs.js
@@ -173,22 +173,40 @@ function _listSubmodules(mirrordir, mirror, keytype, uri, branch, cancellable) {
function ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
params) {
- params = Params.parse(params, {fetch: false,
- fetchKeepGoing: false});
+ params = Params.parse(params, { fetch: false,
+ fetchKeepGoing: false,
+ timeoutSec: 0 });
+ let fetch = params.fetch;
let mirror = getMirrordir(mirrordir, keytype, uri);
let tmpMirror = mirror.get_parent().get_child(mirror.get_basename() + '.tmp');
let didUpdate = false;
let lastFetchPath = getLastfetchPath(mirrordir, keytype, uri, branch);
let lastFetchContents = null;
- if (lastFetchPath.query_exists(cancellable)) {
+ let currentTime = GLib.DateTime.new_now_utc();
+ let lastFetchContents = null;
+ let lastFetchInfo = null;
+ try {
+ lastFetchInfo = lastFetchPath.query_info('time::modified', Gio.FileQueryInfoFlags.NONE, cancellable);
+ } catch (e) {
+ if (!e.matches(Gio.IOErrorEnum, Gio.IOErrorEnum.NOT_FOUND))
+ throw e;
+ }
+ if (lastFetchInfo != null) {
lastFetchContents = GSystem.file_load_contents_utf8(lastFetchPath, cancellable).replace(/[ \n]/g, '');
+ if (params.timeoutSec > 0) {
+ let lastFetchTime = GLib.DateTime.new_from_unix_local(lastFetchInfo.get_attribute_uint64('time::modified'));
+ let diff = currentTime.difference(lastFetchTime) / 1000 / 1000;
+ if (diff < params.timeoutSec) {
+ fetch = false;
+ }
+ }
}
GSystem.shutil_rm_rf(tmpMirror, cancellable);
if (!mirror.query_exists(cancellable)) {
ProcUtil.runSync(['git', 'clone', '--mirror', uri, tmpMirror.get_path()], cancellable);
ProcUtil.runSync(['git', 'config', 'gc.auto', '0'], cancellable, {cwd: tmpMirror});
GSystem.file_rename(tmpMirror, mirror, cancellable);
- } else if (params.fetch) {
+ } else if (fetch) {
try {
ProcUtil.runSync(['git', 'fetch'], cancellable, {cwd:mirror});
} catch (e) {
@@ -210,17 +228,24 @@ function ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
});
}
- if (changed) {
+ if (changed || (fetch && params.timeoutSec > 0)) {
lastFetchPath.replace_contents(currentVcsVersion, null, false, 0, cancellable);
}
return mirror;
}
+function uncacheRepository(mirrordir, keytype, uri, branch, cancellable) {
+ let lastFetchPath = getLastfetchPath(mirrordir, keytype, uri, branch);
+ GSystem.shutil_rm_rf(lastFetchPath, cancellable);
+}
+
function fetch(mirrordir, keytype, uri, branch, cancellable, params) {
- params = Params.parse(params, {keepGoing: false});
+ params = Params.parse(params, {keepGoing: false, timeoutSec: 0});
ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
- {fetch:true, fetchKeepGoing: params.keepGoing});
+ { fetch:true,
+ fetchKeepGoing: params.keepGoing,
+ timeoutSec: params.timeoutSec });
}
function describeVersion(dirpath, branch) {
diff --git a/src/ostbuild/ostbuild.in b/src/ostbuild/ostbuild.in
index 01d75bc..258d9d0 100755
--- a/src/ostbuild/ostbuild.in
+++ b/src/ostbuild/ostbuild.in
@@ -25,4 +25,4 @@ export GIO_USE_VFS=local
export OSTBUILD_DATADIR= pkgdatadir@
export OSTBUILD_LIBDIR= pkglibdir@
-exec gjs -I "${jsdir}" "${jsdir}/main.js" "$@"
+exec $OSTBUILD_GDB gjs -I "${jsdir}" "${jsdir}/main.js" "$@"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
