[Notes] [Git][BuildStream/buildstream][juerg/command-batching] 5 commits: sandbox: Add queue() and run_queue() methods for command batching

[Jürg Billeter <gitlab mg gitlab com>]

Title: GitLab

Jürg Billeter pushed to branch juerg/command-batching at BuildStream / buildstream

Commits:

• bf5c26ff
  by Jürg Billeter at 2018-10-01T08:29:59Z

  sandbox: Add queue() and run_queue() methods for command batching
  

• faa03f51
  by Jürg Billeter at 2018-10-01T08:29:59Z

  buildelement.py: Use Sandbox queue API
  

• 0b6ad713
  by Jürg Billeter at 2018-10-01T08:31:16Z

  WIP: sandbox/_sandboxremote.py: Implement command batching
  

• fd60f4d4
  by Jürg Billeter at 2018-10-01T08:31:16Z

  element.py: Add TODO to __prepare()
  

• 21ec09a3
  by Jürg Billeter at 2018-10-01T08:31:16Z

  element.py: Add TODO to integrate()
  

4 changed files:

• buildstream/buildelement.py
• buildstream/element.py
• buildstream/sandbox/_sandboxremote.py
• buildstream/sandbox/sandbox.py

Changes:

buildstream/buildelement.py

---

...	...	@@ -186,7 +186,9 @@ class BuildElement(Element):
186	186
187	187	with self.timed_activity("Running {}".format(command_name)):
188	188	for cmd in commands:
189		- self.__run_command(sandbox, cmd, command_name)
	189	+ self.__queue_command(sandbox, cmd, command_name)
	190	+
	191	+ sandbox.run_queue(SandboxFlags.ROOT_READ_ONLY)
190	192
191	193	# %{install-root}/%{build-root} should normally not be written
192	194	# to - if an element later attempts to stage to a location
...	...	@@ -210,7 +212,7 @@ class BuildElement(Element):
210	212	if commands:
211	213	with self.timed_activity("Running configure-commands"):
212	214	for cmd in commands:
213		- self.__run_command(sandbox, cmd, 'configure-commands')
	215	+ self.__queue_command(sandbox, cmd, 'configure-commands')
214	216
215	217	def generate_script(self):
216	218	script = ""
...	...	@@ -235,14 +237,18 @@ class BuildElement(Element):
235	237
236	238	return commands
237	239
238		- def __run_command(self, sandbox, cmd, cmd_name):
239		- self.status("Running {}".format(cmd_name), detail=cmd)
	240	+ def __queue_command(self, sandbox, cmd, cmd_name):
	241	+ def start_cb():
	242	+ self.status("Running {}".format(cmd_name), detail=cmd)
	243	+
	244	+ def complete_cb(exitcode):
	245	+ if exitcode != 0:
	246	+ raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
	247	+ collect=self.get_variable('install-root'))
240	248
241	249	# Note the -e switch to 'sh' means to exit with an error
242	250	# if any untested command fails.
243	251	#
244		- exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
245		- SandboxFlags.ROOT_READ_ONLY)
246		- if exitcode != 0:
247		- raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
248		- collect=self.get_variable('install-root'))
	252	+ sandbox.queue(['sh', '-c', '-e', cmd + '\n'],
	253	+ start_callback=start_cb,
	254	+ complete_callback=complete_cb)

buildstream/element.py

---

...	...	@@ -767,6 +767,8 @@ class Element(Plugin):
767	767	bstdata = self.get_public_data('bst')
768	768	environment = self.get_environment()
769	769
	770	+ # TODO support command batching
	771	+
770	772	if bstdata is not None:
771	773	commands = self.node_get_member(bstdata, list, 'integration-commands', [])
772	774	for i in range(len(commands)):
...	...	@@ -2083,6 +2085,8 @@ class Element(Plugin):
2083	2085	self.prepare(sandbox)
2084	2086
2085	2087	if workspace:
	2088	+ # TODO if sandbox._has_queued_commands(): defer/queue the prepared = True
	2089	+ # to avoid setting it prematurely
2086	2090	workspace.prepared = True
2087	2091
2088	2092	def __is_cached(self, keystrength):

buildstream/sandbox/_sandboxremote.py

---

...	...	@@ -19,6 +19,7 @@
19	19	# Jim MacArthur <jim macarthur codethink co uk>
20	20
21	21	import os
	22	+import shlex
22	23	from urllib.parse import urlparse
23	24
24	25	import grpc
...	...	@@ -227,3 +228,38 @@ class SandboxRemote(Sandbox):
227	228	self.process_job_output(action_result.output_directories, action_result.output_files)
228	229
229	230	return 0
	231	+
	232	+ def run_queue(self, flags, *, cwd=None, env=None):
	233	+ queue = self._queue
	234	+ self._queue = []
	235	+
	236	+ script = ""
	237	+ i = 0
	238	+ for entry in queue:
	239	+ cmdline = ' '.join(shlex.quote(cmd) for cmd in entry.command)
	240	+ script += "({})\n".format(cmdline)
	241	+ script += "RETVAL=$?\n"
	242	+ script += "if [ $RETVAL -ne 0 ] ; then\n"
	243	+ # Report failing command and exit code to stderr (and then back to client)
	244	+ script += " echo -e '\nbst-command-failure:' {} $RETVAL >&2\n".format(i)
	245	+ script += " exit 1\n"
	246	+ script += "fi\n"
	247	+ i += 1
	248	+
	249	+ exit_code = self.run(['sh', '-c', script], flags, cwd=cwd, env=env)
	250	+
	251	+ if exit_code != 0:
	252	+ # TODO get failed command and exit code from stderr
	253	+ failed_command = 0
	254	+ command_exit_code = 1
	255	+
	256	+ i = 0
	257	+ for entry in queue:
	258	+ entry.start_callback()
	259	+ if exit_code == 0 or i < failed_command:
	260	+ # Command succeeded
	261	+ entry.complete_callback(0)
	262	+ else:
	263	+ # Command failed
	264	+ entry.complete_callback(command_exit_code)
	265	+ break

buildstream/sandbox/sandbox.py

---

...	...	@@ -29,6 +29,8 @@ See also: :ref:`sandboxing`.
29	29	"""
30	30
31	31	import os
	32	+from collections import namedtuple
	33	+
32	34	from .._exceptions import ImplError, BstError
33	35	from ..storage._filebaseddirectory import FileBasedDirectory
34	36	from ..storage._casbaseddirectory import CasBasedDirectory
...	...	@@ -114,6 +116,9 @@ class Sandbox():
114	116	# directory via get_directory.
115	117	self._never_cache_vdirs = False
116	118
	119	+ # Queued commands
	120	+ self._queue = []
	121	+
117	122	def get_directory(self):
118	123	"""Fetches the sandbox root directory
119	124
...	...	@@ -228,6 +233,53 @@ class Sandbox():
228	233	raise ImplError("Sandbox of type '{}' does not implement run()"
229	234	.format(type(self).__name__))
230	235
	236	+ def queue(self, command, *, start_callback=None, complete_callback=None):
	237	+ """Queue a command to be run in the sandbox.
	238	+
	239	+ If the command fails, commands queued later will not be executed.
	240	+ The callbacks are not guaranteed to be invoked in real time.
	241	+
	242	+ Args:
	243	+ command (list): The command to run in the sandboxed environment, as a list
	244	+ of strings starting with the binary to run.
	245	+ start_callback (callable): Called when the command starts.
	246	+ complete_callback (callble): Called when the command completes
	247	+ with the exit code as argument.
	248	+ """
	249	+ entry = namedtuple('QueueEntry', ['command', 'start_callback', 'complete_callback'])
	250	+ entry.command = command
	251	+ entry.start_callback = start_callback
	252	+ entry.complete_callback = complete_callback
	253	+ self._queue.append(entry)
	254	+
	255	+ def run_queue(self, flags, *, cwd=None, env=None):
	256	+ """Run a command in the sandbox.
	257	+
	258	+ Args:
	259	+ flags (:class:`.SandboxFlags`): The flags for running this command.
	260	+ cwd (str): The sandbox relative working directory in which to run the command.
	261	+ env (dict): A dictionary of string key, value pairs to set as environment
	262	+ variables inside the sandbox environment.
	263	+
	264	+ Raises:
	265	+ (:class:`.ProgramNotFoundError`): If a host tool which the given sandbox
	266	+ implementation requires is not found.
	267	+
	268	+ .. note::
	269	+
	270	+ The optional *cwd* argument will default to the value set with
	271	+ :func:`~buildstream.sandbox.Sandbox.set_work_directory`
	272	+ """
	273	+ queue = self._queue
	274	+ self._queue = []
	275	+
	276	+ for entry in queue:
	277	+ entry.start_callback()
	278	+ exit_code = self.run(entry.command, flags, cwd=cwd, env=env)
	279	+ entry.complete_callback(exit_code)
	280	+ if exit_code != 0:
	281	+ break
	282	+
231	283	################################################
232	284	# Private methods #
233	285	################################################

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.