[Notes] [Git][BuildStream/buildstream][tpollard/workspacebuildtree] 4 commits: .gitlab-ci.yml: Use new testsuite images with pycodestyle 1.4.0

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream

Commits:

• 3b5c8a28
  by Tristan Maat at 2019-01-02T15:57:13Z

  .gitlab-ci.yml: Use new testsuite images with pycodestyle 1.4.0
  
  We need to bump the testsuite image versions to use the new
  pycodestyle update that allows excluding file paths, so that we can
  exclude generated python code.
  

• bd60e8a9
  by Javier Jardón at 2019-01-02T16:59:41Z

  Use pycodestyle instead of pep8
  
  pep8 package was renamed to pycodestyle to reduce confusion.
  
  Disabled warnings:
  W504 - Line break after binary operator (seems people like this)
  W605 - Invalid escape sequence (some of our regexes use these)
  

• 32c47d1c
  by Tristan Maat at 2019-01-02T17:32:09Z

  Merge branch 'jjardon/pycodestyle' into 'master'
  
  Use pycodestyle instead pep8 python module
  
  See merge request BuildStream/buildstream!638

• f67a16e3
  by Tom Pollard at 2019-01-03T10:59:40Z

  WIP: Opening a workspace with a cached build
  

18 changed files:

• .gitlab-ci.yml
• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_gitsourcebase.py
• buildstream/_ostree.py
• buildstream/_signals.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/element.py
• buildstream/source.py
• dev-requirements.txt
• doc/source/developing/workspaces.rst
• setup.cfg
• tests/frontend/workspace.py
• tests/integration/workspace.py
• tests/testutils/http_server.py

Changes:

.gitlab-ci.yml

---

1		-image: buildstream/testsuite-debian:9-master-123-7ce6581b
	1	+image: buildstream/testsuite-debian:9-06bab030-b848172c
2	2
3	3	cache:
4	4	key: "$CI_JOB_NAME-"
...	...	@@ -91,23 +91,23 @@ source_dist:
91	91	- ${COVERAGE_DIR}
92	92
93	93	tests-debian-9:
94		- image: buildstream/testsuite-debian:9-master-123-7ce6581b
	94	+ image: buildstream/testsuite-debian:9-06bab030-b848172c
95	95	<<: *tests
96	96
97	97	tests-fedora-27:
98		- image: buildstream/testsuite-fedora:27-master-123-7ce6581b
	98	+ image: buildstream/testsuite-fedora:27-06bab030-b848172c
99	99	<<: *tests
100	100
101	101	tests-fedora-28:
102		- image: buildstream/testsuite-fedora:28-master-123-7ce6581b
	102	+ image: buildstream/testsuite-fedora:28-06bab030-b848172c
103	103	<<: *tests
104	104
105	105	tests-ubuntu-18.04:
106		- image: buildstream/testsuite-ubuntu:18.04-master-123-7ce6581b
	106	+ image: buildstream/testsuite-ubuntu:18.04-06bab030-b848172c
107	107	<<: *tests
108	108
109	109	overnight-fedora-28-aarch64:
110		- image: buildstream/testsuite-fedora:aarch64-28-master-123-7ce6581b
	110	+ image: buildstream/testsuite-fedora:aarch64-28-06bab030-32a101f6
111	111	tags:
112	112	- aarch64
113	113	<<: *tests
...	...	@@ -120,7 +120,7 @@ overnight-fedora-28-aarch64:
120	120	tests-unix:
121	121	# Use fedora here, to a) run a test on fedora and b) ensure that we
122	122	# can get rid of ostree - this is not possible with debian-8
123		- image: buildstream/testsuite-fedora:27-master-123-7ce6581b
	123	+ image: buildstream/testsuite-fedora:27-06bab030-b848172c
124	124	<<: *tests
125	125	variables:
126	126	BST_FORCE_BACKEND: "unix"
...	...	@@ -140,7 +140,7 @@ tests-unix:
140	140
141	141	tests-fedora-missing-deps:
142	142	# Ensure that tests behave nicely while missing bwrap and ostree
143		- image: buildstream/testsuite-fedora:28-master-123-7ce6581b
	143	+ image: buildstream/testsuite-fedora:28-06bab030-b848172c
144	144	<<: *tests
145	145
146	146	script:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -766,6 +766,20 @@ class ArtifactCache():
766	766
767	767	self.cas.link_ref(oldref, newref)
768	768
	769	+ # checkout_artifact_subdir()
	770	+ #
	771	+ # Checkout given artifact subdir into provided directory
	772	+ #
	773	+ # Args:
	774	+ # element (Element): The Element
	775	+ # key (str): The cache key to use
	776	+ # subdir (str): The subdir to checkout
	777	+ # tmpdir (str): The dir to place the subdir content
	778	+ #
	779	+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
	780	+ ref = self.get_artifact_fullname(element, key)
	781	+ return self.cas.checkout_artifact_subdir(ref, subdir, tmpdir)
	782	+
769	783	################################################
770	784	# Local Private Methods #
771	785	################################################

buildstream/_artifactcache/cascache.py

---

...	...	@@ -459,6 +459,21 @@ class CASCache():
459	459
460	460	return True
461	461
	462	+ # checkout_artifact_subdir():
	463	+ #
	464	+ # Checkout given artifact subdir into provided directory
	465	+ #
	466	+ # Args:
	467	+ # ref (str): The ref to check
	468	+ # subdir (str): The subdir to checkout
	469	+ # tmpdir (str): The dir to place the subdir content
	470	+ #
	471	+ def checkout_artifact_subdir(self, ref, subdir, tmpdir):
	472	+ tree = self.resolve_ref(ref)
	473	+ # This assumes that the subdir digest is present in the element tree
	474	+ subdirdigest = self._get_subdir(tree, subdir)
	475	+ self._checkout(tmpdir, subdirdigest)
	476	+
462	477	# objpath():
463	478	#
464	479	# Return the path of an object based on its digest.

buildstream/_context.py

---

...	...	@@ -129,6 +129,9 @@ class Context():
129	129	# a hard reset of a workspace, potentially losing changes.
130	130	self.prompt_workspace_reset_hard = None
131	131
	132	+ # Whether to not include artifact buildtrees in workspaces if available
	133	+ self.workspace_buildtrees = True
	134	+
132	135	# Whether elements must be rebuilt when their dependencies have changed
133	136	self._strict_build_plan = None
134	137
...	...	@@ -187,7 +190,7 @@ class Context():
187	190	_yaml.node_validate(defaults, [
188	191	'sourcedir', 'builddir', 'artifactdir', 'logdir',
189	192	'scheduler', 'artifacts', 'logging', 'projects',
190		- 'cache', 'prompt', 'workspacedir',
	193	+ 'cache', 'prompt', 'workspacedir', 'workspace-buildtrees'
191	194	])
192	195
193	196	for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
...	...	@@ -215,6 +218,9 @@ class Context():
215	218	# Load pull build trees configuration
216	219	self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
217	220
	221	+ # Load workspace buildtrees configuration
	222	+ self.workspace_buildtrees = _yaml.node_get(defaults, bool, 'workspace-buildtrees', default_value='True')
	223	+
218	224	# Load logging config
219	225	logging = _yaml.node_get(defaults, Mapping, 'logging')
220	226	_yaml.node_validate(logging, [

buildstream/_frontend/cli.py

---

...	...	@@ -828,7 +828,7 @@ def workspace():
828	828	##################################################################
829	829	@workspace.command(name='open', short_help="Open a new workspace")
830	830	@click.option('--no-checkout', default=False, is_flag=True,
831		- help="Do not checkout the source, only link to the given directory")
	831	+ help="Do not checkout the source or cached buildtree, only link to the given directory")
832	832	@click.option('--force', '-f', default=False, is_flag=True,
833	833	help="The workspace will be created even if the directory in which it will be created is not empty " +
834	834	"or if a workspace for that element already exists")
...	...	@@ -837,16 +837,25 @@ def workspace():
837	837	@click.option('--directory', type=click.Path(file_okay=False), default=None,
838	838	help="Only for use when a single Element is given: Set the directory to use to create the workspace")
839	839	@click.argument('elements', nargs=-1, type=click.Path(readable=False), required=True)
	840	+@click.option('--no-cache', default=False, is_flag=True,
	841	+ help="Do not checkout the cached buildtree")
840	842	@click.pass_obj
841		-def workspace_open(app, no_checkout, force, track_, directory, elements):
842		- """Open a workspace for manual source modification"""
	843	+def workspace_open(app, no_checkout, force, track_, directory, elements, no_cache):
	844	+
	845	+ """Open a workspace for manual source modification, the elements buildtree
	846	+ will be provided if available in the local artifact cache.
	847	+ """
	848	+
	849	+ if not no_cache and not no_checkout:
	850	+ click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
843	851
844	852	with app.initialized():
845	853	app.stream.workspace_open(elements,
846	854	no_checkout=no_checkout,
847	855	track_first=track_,
848	856	force=force,
849		- custom_dir=directory)
	857	+ custom_dir=directory,
	858	+ no_cache=no_cache)
850	859
851	860
852	861	##################################################################

buildstream/_gitsourcebase.py

---

...	...	@@ -222,6 +222,31 @@ class GitMirror(SourceFetcher):
222	222	fail="Failed to checkout git ref {}".format(self.ref),
223	223	cwd=fullpath)
224	224
	225	+ def init_cached_build_workspace(self, directory):
	226	+ fullpath = os.path.join(directory, self.path)
	227	+ url = self.source.translate_url(self.url)
	228	+
	229	+ self.source.call([self.source.host_git, 'init', fullpath],
	230	+ fail="Failed to init git in directory: {}".format(fullpath),
	231	+ fail_temporarily=True,
	232	+ cwd=fullpath)
	233	+
	234	+ self.source.call([self.source.host_git, 'fetch', self.mirror],
	235	+ fail='Failed to fetch from local mirror "{}"'.format(self.mirror),
	236	+ cwd=fullpath)
	237	+
	238	+ self.source.call([self.source.host_git, 'remote', 'add', 'origin', url],
	239	+ fail='Failed to add remote origin "{}"'.format(url),
	240	+ cwd=fullpath)
	241	+
	242	+ self.source.call([self.source.host_git, 'update-ref', '--no-deref', 'HEAD', self.ref],
	243	+ fail='Failed update HEAD to ref "{}"'.format(self.ref),
	244	+ cwd=fullpath)
	245	+
	246	+ self.source.call([self.source.host_git, 'read-tree', 'HEAD'],
	247	+ fail='Failed to read HEAD into index',
	248	+ cwd=fullpath)
	249	+
225	250	# List the submodules (path/url tuples) present at the given ref of this repo
226	251	def submodule_list(self):
227	252	modules = "{}:{}".format(self.ref, GIT_MODULES)
...	...	@@ -515,6 +540,14 @@ class _GitSourceBase(Source):
515	540	for mirror in self.submodules:
516	541	mirror.init_workspace(directory)
517	542
	543	+ def init_cached_build_workspace(self, directory):
	544	+ self._refresh_submodules()
	545	+
	546	+ with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
	547	+ self.mirror.init_cached_build_workspace(directory)
	548	+ for mirror in self.submodules:
	549	+ mirror.init_cached_build_workspace(directory)
	550	+
518	551	def stage(self, directory):
519	552
520	553	# Need to refresh submodule list here again, because

buildstream/_ostree.py

---

...	...	@@ -34,7 +34,7 @@ from ._exceptions import BstError, ErrorDomain
34	34
35	35	# pylint: disable=wrong-import-position,wrong-import-order
36	36	gi.require_version('OSTree', '1.0')
37		-from gi.repository import GLib, Gio, OSTree # nopep8
	37	+from gi.repository import GLib, Gio, OSTree # noqa
38	38
39	39
40	40	# For users of this file, they must expect (except) it.

buildstream/_signals.py

---

...	...	@@ -38,7 +38,7 @@ def terminator_handler(signal_, frame):
38	38	terminator_ = terminator_stack.pop()
39	39	try:
40	40	terminator_()
41		- except: # pylint: disable=bare-except
	41	+ except: # noqa pylint: disable=bare-except
42	42	# Ensure we print something if there's an exception raised when
43	43	# processing the handlers. Note that the default exception
44	44	# handler won't be called because we os._exit next, so we must

buildstream/_stream.py

---

...	...	@@ -489,14 +489,21 @@ class Stream():
489	489	# track_first (bool): Whether to track and fetch first
490	490	# force (bool): Whether to ignore contents in an existing directory
491	491	# custom_dir (str): Custom location to create a workspace or false to use default location.
	492	+ # no_cache (bool): Whether to not include the cached buildtree
492	493	#
493	494	def workspace_open(self, targets, *,
494	495	no_checkout,
495	496	track_first,
496	497	force,
497		- custom_dir):
	498	+ custom_dir,
	499	+ no_cache):
	500	+
498	501	# This function is a little funny but it is trying to be as atomic as possible.
499	502
	503	+ # Set no_cache if the global user conf workspacebuildtrees is false
	504	+ if not self._context.workspace_buildtrees:
	505	+ no_cache = True
	506	+
500	507	if track_first:
501	508	track_targets = targets
502	509	else:
...	...	@@ -554,7 +561,7 @@ class Stream():
554	561	directory = os.path.abspath(custom_dir)
555	562	expanded_directories = [directory, ]
556	563	else:
557		- # If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the
	564	+ # If this fails it is a bug in whatever calls this, usually cli.py and so can not be tested for via the
558	565	# run bst test mechanism.
559	566	assert len(elements) == len(expanded_directories)
560	567
...	...	@@ -569,12 +576,26 @@ class Stream():
569	576	.format(target.name, directory), reason='bad-directory')
570	577
571	578	# So far this function has tried to catch as many issues as possible with out making any changes
572		- # Now it dose the bits that can not be made atomic.
	579	+ # Now it does the bits that can not be made atomic.
573	580	targetGenerator = zip(elements, expanded_directories)
574	581	for target, directory in targetGenerator:
575	582	self._message(MessageType.INFO, "Creating workspace for element {}"
576	583	.format(target.name))
577	584
	585	+ # Check if given target has a buildtree artifact cached locally
	586	+ buildtree = None
	587	+ if target._cached():
	588	+ buildtree = target._cached_buildtree()
	589	+
	590	+ # If we're running in the default state, make the user aware of buildtree usage
	591	+ if not no_cache and not no_checkout:
	592	+ if buildtree:
	593	+ self._message(MessageType.INFO, "{} buildtree artifact is available,"
	594	+ " workspace will be opened with it".format(target.name))
	595	+ else:
	596	+ self._message(MessageType.WARN, "{} buildtree artifact not available,"
	597	+ " workspace will be opened with source checkout".format(target.name))
	598	+
578	599	workspace = workspaces.get_workspace(target._get_full_name())
579	600	if workspace:
580	601	workspaces.delete_workspace(target._get_full_name())
...	...	@@ -589,7 +610,20 @@ class Stream():
589	610	todo_elements = "\nDid not try to create workspaces for " + todo_elements
590	611	raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
591	612
592		- workspaces.create_workspace(target, directory, checkout=not no_checkout)
	613	+ # Handle opening workspace with buildtree included
	614	+ if (buildtree and not no_cache) and not no_checkout:
	615	+ workspaces.create_workspace(target, directory, checkout=not no_checkout, cached_build=buildtree)
	616	+ with target.timed_activity("Staging buildtree to {}".format(directory)):
	617	+ target._open_workspace(buildtree=buildtree)
	618	+ else:
	619	+ workspaces.create_workspace(target, directory, checkout=not no_checkout)
	620	+ if (not buildtree or no_cache) and not no_checkout:
	621	+ with target.timed_activity("Staging sources to {}".format(directory)):
	622	+ target._open_workspace()
	623	+
	624	+ # Saving the workspace once it is set up means that if the next workspace fails to be created before
	625	+ # the configuration gets saved. The successfully created workspace still gets saved.
	626	+ workspaces.save_config()
593	627	self._message(MessageType.INFO, "Created a workspace for element: {}"
594	628	.format(target._get_full_name()))
595	629
...	...	@@ -672,7 +706,25 @@ class Stream():
672	706	.format(workspace_path, e)) from e
673	707
674	708	workspaces.delete_workspace(element._get_full_name())
675		- workspaces.create_workspace(element, workspace_path, checkout=True)
	709	+
	710	+ # Create the workspace, ensuring the original optional cached build state is preserved if
	711	+ # possible.
	712	+ buildtree = False
	713	+ if workspace.cached_build and element._cached():
	714	+ if self._artifacts.contains_subdir_artifact(element, element._get_cache_key(), 'buildtree'):
	715	+ buildtree = True
	716	+
	717	+ # Warn the user if the workspace cannot be opened with the original cached build state
	718	+ if workspace.cached_build and not buildtree:
	719	+ self._message(MessageType.WARN, "{} original buildtree artifact not available,"
	720	+ " workspace will be opened with source checkout".format(element.name))
	721	+
	722	+ # If opening the cached build, set checkout to false
	723	+ workspaces.create_workspace(element, workspace_path,
	724	+ checkout=not buildtree, cached_build=buildtree)
	725	+
	726	+ with element.timed_activity("Staging to {}".format(workspace_path)):
	727	+ element._open_workspace(buildtree=buildtree)
676	728
677	729	self._message(MessageType.INFO,
678	730	"Reset workspace for {} at: {}".format(element.name,

buildstream/_workspaces.py

---

...	...	@@ -24,7 +24,7 @@ from . import _yaml
24	24	from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27		-BST_WORKSPACE_FORMAT_VERSION = 3
	27	+BST_WORKSPACE_FORMAT_VERSION = 4
28	28	BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1
29	29	WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
30	30
...	...	@@ -239,9 +239,11 @@ class WorkspaceProjectCache():
239	239	# running_files (dict): A dict mapping dependency elements to files
240	240	# changed between failed builds. Should be
241	241	# made obsolete with failed build artifacts.
	242	+# cached_build (bool): If the workspace is staging the cached build artifact
242	243	#
243	244	class Workspace():
244		- def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
	245	+ def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False,
	246	+ running_files=None, cached_build=False):
245	247	self.prepared = prepared
246	248	self.last_successful = last_successful
247	249	self._path = path
...	...	@@ -249,6 +251,7 @@ class Workspace():
249	251
250	252	self._toplevel_project = toplevel_project
251	253	self._key = None
	254	+ self.cached_build = cached_build
252	255
253	256	# to_dict()
254	257	#
...	...	@@ -261,7 +264,8 @@ class Workspace():
261	264	ret = {
262	265	'prepared': self.prepared,
263	266	'path': self._path,
264		- 'running_files': self.running_files
	267	+ 'running_files': self.running_files,
	268	+ 'cached_build': self.cached_build
265	269	}
266	270	if self.last_successful is not None:
267	271	ret["last_successful"] = self.last_successful
...	...	@@ -429,8 +433,9 @@ class Workspaces():
429	433	# target (Element) - The element to create a workspace for
430	434	# path (str) - The path in which the workspace should be kept
431	435	# checkout (bool): Whether to check-out the element's sources into the directory
	436	+ # cached_build (bool) - If the workspace is staging the cached build artifact
432	437	#
433		- def create_workspace(self, target, path, *, checkout):
	438	+ def create_workspace(self, target, path, *, checkout, cached_build=False):
434	439	element_name = target._get_full_name()
435	440	project_dir = self._toplevel_project.directory
436	441	if path.startswith(project_dir):
...	...	@@ -438,7 +443,8 @@ class Workspaces():
438	443	else:
439	444	workspace_path = path
440	445
441		- self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path)
	446	+ self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path,
	447	+ cached_build=cached_build)
442	448
443	449	if checkout:
444	450	with target.timed_activity("Staging sources to {}".format(path)):
...	...	@@ -627,6 +633,7 @@ class Workspaces():
627	633	'path': _yaml.node_get(node, str, 'path'),
628	634	'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
629	635	'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
	636	+ 'cached_build': _yaml.node_get(node, bool, 'cached_build', default_value=False)
630	637	}
631	638	return Workspace.from_dict(self._toplevel_project, dictionary)
632	639

buildstream/element.py

---

...	...	@@ -1568,7 +1568,7 @@ class Element(Plugin):
1568	1568	utils._force_rmtree(rootdir)
1569	1569
1570	1570	with _signals.terminator(cleanup_rootdir), \
1571		- self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # nopep8
	1571	+ self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # noqa
1572	1572
1573	1573	# By default, the dynamic public data is the same as the static public data.
1574	1574	# The plugin's assemble() method may modify this, though.
...	...	@@ -1912,7 +1912,10 @@ class Element(Plugin):
1912	1912	# This requires that a workspace already be created in
1913	1913	# the workspaces metadata first.
1914	1914	#
1915		- def _open_workspace(self):
	1915	+ # Args:
	1916	+ # buildtree (bool): Whether to open workspace with artifact buildtree
	1917	+ #
	1918	+ def _open_workspace(self, buildtree=False):
1916	1919	context = self._get_context()
1917	1920	workspace = self._get_workspace()
1918	1921	assert workspace is not None
...	...	@@ -1925,11 +1928,19 @@ class Element(Plugin):
1925	1928	# files in the target directory actually works without any
1926	1929	# additional support from Source implementations.
1927	1930	#
	1931	+
1928	1932	os.makedirs(context.builddir, exist_ok=True)
1929	1933	with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
1930	1934	.format(self.normal_name)) as temp:
1931		- for source in self.sources():
1932		- source._init_workspace(temp)
	1935	+
	1936	+ # Checkout cached buildtree, augment with source plugin if applicable
	1937	+ if buildtree:
	1938	+ self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', temp)
	1939	+ for source in self.sources():
	1940	+ source._init_cached_build_workspace(temp)
	1941	+ else:
	1942	+ for source in self.sources():
	1943	+ source._init_workspace(temp)
1933	1944
1934	1945	# Now hardlink the files into the workspace target.
1935	1946	utils.link_files(temp, workspace.get_absolute_path())

buildstream/source.py

---

...	...	@@ -465,6 +465,24 @@ class Source(Plugin):
465	465	"""
466	466	self.stage(directory)
467	467
	468	+ def init_cached_build_workspace(self, directory):
	469	+ """Initialises a new cached build workspace
	470	+
	471	+ Args:
	472	+ directory (str): Path of the workspace to init
	473	+
	474	+ Raises:
	475	+ :class:`.SourceError`
	476	+
	477	+ Implementors overriding this method should assume that *directory*
	478	+ already exists.
	479	+
	480	+ Implementors should raise :class:`.SourceError` when encountering
	481	+ some system error.
	482	+ """
	483	+ # Allow a non implementation
	484	+ return None
	485	+
468	486	def get_source_fetchers(self):
469	487	"""Get the objects that are used for fetching
470	488
...	...	@@ -717,6 +735,12 @@ class Source(Plugin):
717	735
718	736	self.init_workspace(directory)
719	737
	738	+ # Wrapper for init_cached_build_workspace()
	739	+ def _init_cached_build_workspace(self, directory):
	740	+ directory = self.__ensure_directory(directory)
	741	+
	742	+ self.init_cached_build_workspace(directory)
	743	+
720	744	# _get_unique_key():
721	745	#
722	746	# Wrapper for get_unique_key() api

dev-requirements.txt

---

1	1	coverage == 4.4.0
2		-pep8
3	2	pylint
4	3	pytest >= 3.9
	4	+pytest-codestyle >= 1.4.0
5	5	pytest-cov >= 2.5.0
6	6	pytest-datafiles >= 2.0
7	7	pytest-env
8		-pytest-pep8
9	8	pytest-pylint
10	9	pytest-xdist
11	10	pytest-timeout

doc/source/developing/workspaces.rst

---

...	...	@@ -24,9 +24,32 @@ Suppose we now want to alter the functionality of the *hello* command. We can
24	24	make changes to the source code of Buildstream elements by making use of
25	25	BuildStream's workspace command.
26	26
	27	+Utilising cached buildtrees
	28	+---------------------------
	29	+ When a BuildStream build element artifact is created and cached, a snapshot of
	30	+ the build directory after the build commands have completed is included in the
	31	+ artifact. This `build tree` can be considered an intermediary state of element,
	32	+ where the source is present along with any output created during the build
	33	+ execution.
	34	+
	35	+ By default when opening a workspace, bst will attempt to stage the build tree
	36	+ into the workspace if it's available in the local cache. If the respective
	37	+ build tree is not present in the cache (element not cached, partially cached or
	38	+ is a non build element) then the source will be staged as is. The default
	39	+ behaviour to attempt to use the build tree can be overriden with specific bst
	40	+ workspace open option of `--no-cache`, or via setting user configuration option
	41	+ `workspacebuildtrees: False`
	42	+
27	43
28	44	Opening a workspace
29	45	-------------------
	46	+.. note::
	47	+
	48	+ This example presumes you built the hello.bst during
	49	+ :ref:`running commands <tutorial_running_commands>`
	50	+ if not, please start by building it.
	51	+
	52	+
30	53	First we need to open a workspace, we can do this by running
31	54
32	55	.. raw:: html
...	...	@@ -93,6 +116,15 @@ Alternatively, if we wish to discard the changes we can use
93	116
94	117	This resets the workspace to its original state.
95	118
	119	+.. note::
	120	+
	121	+ bst reset will attempt to open the workspace in
	122	+ the condition in which it was originally staged,
	123	+ i.e with or without consuming the element build tree.
	124	+ If it was originally staged with a cached build tree
	125	+ and there's no longer one available, the source will
	126	+ be staged as is.
	127	+
96	128	To discard the workspace completely we can do:
97	129
98	130	.. raw:: html

setup.cfg

---

...	...	@@ -11,20 +11,11 @@ parentdir_prefix = BuildStream-
11	11	test=pytest
12	12
13	13	[tool:pytest]
14		-addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
	14	+addopts = --verbose --basetemp ./tmp --codestyle --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
15	15	norecursedirs = tests/integration/project integration-cache tmp __pycache__ .eggs
16	16	python_files = tests/*/*.py
17		-pep8maxlinelength = 119
18		-pep8ignore =
19		- * E129
20		- * E125
21		- doc/source/conf.py ALL
22		- tmp/* ALL
23		- */lib/python3* ALL
24		- */bin/* ALL
25		- buildstream/_fuse/fuse.py ALL
26		- .eggs/* ALL
27		- *_pb2.py ALL
28		- *_pb2_grpc.py ALL
	17	+codestyle_max_line_length = 119
	18	+codestyle_ignore = E129 E125 W504 W605
	19	+codestyle_exclude = doc/source/conf.py buildstream/_fuse/fuse.py buildstream/_protos/**/*py
29	20	env =
30	21	D:BST_TEST_SUITE=True

tests/frontend/workspace.py

---

...	...	@@ -92,18 +92,18 @@ class WorkspaceCreater():
92	92	element_name))
93	93	return element_name, element_path, workspace_dir
94	94
95		- def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None,
	95	+ def create_workspace_elements(self, kinds, track, suffixes=None, workspace_dir_usr=None,
96	96	element_attrs=None):
97	97
98	98	element_tuples = []
99	99
100		- if suffixs is None:
101		- suffixs = ['', ] * len(kinds)
	100	+ if suffixes is None:
	101	+ suffixes = ['', ] * len(kinds)
102	102	else:
103		- if len(suffixs) != len(kinds):
	103	+ if len(suffixes) != len(kinds):
104	104	raise "terable error"
105	105
106		- for suffix, kind in zip(suffixs, kinds):
	106	+ for suffix, kind in zip(suffixes, kinds):
107	107	element_name, element_path, workspace_dir = \
108	108	self.create_workspace_element(kind, track, suffix, workspace_dir_usr,
109	109	element_attrs)
...	...	@@ -118,10 +118,10 @@ class WorkspaceCreater():
118	118
119	119	return element_tuples
120	120
121		- def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None,
122		- element_attrs=None):
	121	+ def open_workspaces(self, kinds, track, suffixes=None, workspace_dir=None,
	122	+ element_attrs=None, no_cache=False):
123	123
124		- element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir,
	124	+ element_tuples = self.create_workspace_elements(kinds, track, suffixes, workspace_dir,
125	125	element_attrs)
126	126	os.makedirs(self.workspace_cmd, exist_ok=True)
127	127
...	...	@@ -130,12 +130,15 @@ class WorkspaceCreater():
130	130	args = ['workspace', 'open']
131	131	if track:
132	132	args.append('--track')
	133	+ if no_cache:
	134	+ args.append('--no-cache')
133	135	if workspace_dir is not None:
134	136	assert len(element_tuples) == 1, "test logic error"
135	137	_, workspace_dir = element_tuples[0]
136	138	args.extend(['--directory', workspace_dir])
137		-
	139	+ print("element_tuples", element_tuples)
138	140	args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
	141	+ print("args", args)
139	142	result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
140	143
141	144	result.assert_success()
...	...	@@ -149,14 +152,14 @@ class WorkspaceCreater():
149	152	filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
150	153	assert os.path.exists(filename)
151	154
152		- return element_tuples
	155	+ return element_tuples, result
153	156
154	157
155	158	def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
156		- project_path=None, element_attrs=None):
	159	+ project_path=None, element_attrs=None, no_cache=False):
157	160	workspace_object = WorkspaceCreater(cli, tmpdir, datafiles, project_path)
158		- workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
159		- element_attrs)
	161	+ workspaces, _ = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
	162	+ element_attrs, no_cache)
160	163	assert len(workspaces) == 1
161	164	element_name, workspace = workspaces[0]
162	165	return element_name, workspace_object.project_path, workspace
...	...	@@ -190,7 +193,7 @@ def test_open_bzr_customize(cli, tmpdir, datafiles):
190	193	def test_open_multi(cli, tmpdir, datafiles):
191	194
192	195	workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
193		- workspaces = workspace_object.open_workspaces(repo_kinds, False)
	196	+ workspaces, _ = workspace_object.open_workspaces(repo_kinds, False)
194	197
195	198	for (elname, workspace), kind in zip(workspaces, repo_kinds):
196	199	assert kind in elname
...	...	@@ -823,7 +826,9 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
823	826	"alpha.bst": {
824	827	"prepared": False,
825	828	"path": "/workspaces/bravo",
826		- "running_files": {}
	829	+ "running_files": {},
	830	+ "cached_build": False
	831	+
827	832	}
828	833	}
829	834	}),
...	...	@@ -838,7 +843,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
838	843	"alpha.bst": {
839	844	"prepared": False,
840	845	"path": "/workspaces/bravo",
841		- "running_files": {}
	846	+ "running_files": {},
	847	+ "cached_build": False
842	848	}
843	849	}
844	850	}),
...	...	@@ -856,7 +862,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
856	862	"alpha.bst": {
857	863	"prepared": False,
858	864	"path": "/workspaces/bravo",
859		- "running_files": {}
	865	+ "running_files": {},
	866	+ "cached_build": False
860	867	}
861	868	}
862	869	}),
...	...	@@ -881,7 +888,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
881	888	"last_successful": "some_key",
882	889	"running_files": {
883	890	"beta.bst": ["some_file"]
884		- }
	891	+ },
	892	+ "cached_build": False
885	893	}
886	894	}
887	895	}),
...	...	@@ -901,7 +909,30 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
901	909	"alpha.bst": {
902	910	"prepared": True,
903	911	"path": "/workspaces/bravo",
904		- "running_files": {}
	912	+ "running_files": {},
	913	+ "cached_build": False
	914	+ }
	915	+ }
	916	+ }),
	917	+ # Test loading version 4
	918	+ ({
	919	+ "format-version": 4,
	920	+ "workspaces": {
	921	+ "alpha.bst": {
	922	+ "prepared": False,
	923	+ "path": "/workspaces/bravo",
	924	+ "running_files": {},
	925	+ "cached_build": True
	926	+ }
	927	+ }
	928	+ }, {
	929	+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
	930	+ "workspaces": {
	931	+ "alpha.bst": {
	932	+ "prepared": False,
	933	+ "path": "/workspaces/bravo",
	934	+ "running_files": {},
	935	+ "cached_build": True
905	936	}
906	937	}
907	938	})
...	...	@@ -1209,3 +1240,80 @@ def test_external_list(cli, datafiles, tmpdir_factory):
1209	1240
1210	1241	result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
1211	1242	result.assert_success()
	1243	+
	1244	+
	1245	+@pytest.mark.datafiles(DATA_DIR)
	1246	+def test_nocache_open_messages(cli, tmpdir, datafiles):
	1247	+
	1248	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1249	+ _, result = workspace_object.open_workspaces(('git', ), False)
	1250	+
	1251	+ # cli default WARN for source dropback possibility when no-cache flag is not passed
	1252	+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" in result.output
	1253	+
	1254	+ # cli WARN for source dropback happening when no-cache flag not given, but buildtree not available
	1255	+ assert "workspace will be opened with source checkout" in result.stderr
	1256	+
	1257	+ # cli default WARN for source dropback possibilty not given when no-cache flag is passed
	1258	+ tmpdir = os.path.join(str(tmpdir), "2")
	1259	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1260	+ _, result = workspace_object.open_workspaces(('git', ), False, suffixes='1', no_cache=True)
	1261	+
	1262	+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" not in result.output
	1263	+
	1264	+
	1265	+@pytest.mark.datafiles(DATA_DIR)
	1266	+def test_nocache_reset_messages(cli, tmpdir, datafiles):
	1267	+
	1268	+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
	1269	+ workspaces, result = workspace_object.open_workspaces(('git', ), False)
	1270	+ element_name, workspace = workspaces[0]
	1271	+ project = workspace_object.project_path
	1272	+
	1273	+ # Modify workspace, without building so the artifact is not cached
	1274	+ shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
	1275	+ os.makedirs(os.path.join(workspace, 'etc'))
	1276	+ with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
	1277	+ f.write("PONY='pink'")
	1278	+
	1279	+ # Now reset the open workspace, this should have the
	1280	+ # effect of reverting our changes to the original source, as it
	1281	+ # was not originally opened with a cached buildtree and as such
	1282	+ # should not notify the user
	1283	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1284	+ 'workspace', 'reset', element_name
	1285	+ ])
	1286	+ result.assert_success()
	1287	+ assert "original buildtree artifact not available" not in result.output
	1288	+ assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
	1289	+ assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
	1290	+
	1291	+ # Close the workspace
	1292	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1293	+ 'workspace', 'close', '--remove-dir', element_name
	1294	+ ])
	1295	+ result.assert_success()
	1296	+
	1297	+ # Build the workspace so we have a cached buildtree artifact for the element
	1298	+ assert cli.get_element_state(project, element_name) == 'buildable'
	1299	+ result = cli.run(project=project, args=['build', element_name])
	1300	+ result.assert_success()
	1301	+
	1302	+ # Opening the workspace after a build should lead to the cached buildtree being
	1303	+ # staged by default
	1304	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1305	+ 'workspace', 'open', element_name
	1306	+ ])
	1307	+ result.assert_success()
	1308	+
	1309	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1310	+ 'workspace', 'list'
	1311	+ ])
	1312	+ result.assert_success()
	1313	+ # Now reset the workspace and ensure that a warning is not given about the artifact
	1314	+ # buildtree not being available
	1315	+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
	1316	+ 'workspace', 'reset', element_name
	1317	+ ])
	1318	+ result.assert_success()
	1319	+ assert "original buildtree artifact not available" not in result.output

tests/integration/workspace.py

---

...	...	@@ -278,3 +278,39 @@ def test_incremental_configure_commands_run_only_once(cli, tmpdir, datafiles):
278	278	res = cli.run(project=project, args=['build', element_name])
279	279	res.assert_success()
280	280	assert not os.path.exists(os.path.join(workspace, 'prepared-again'))
	281	+
	282	+
	283	+@pytest.mark.integration
	284	+@pytest.mark.datafiles(DATA_DIR)
	285	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	286	+def test_workspace_contains_buildtree(cli, tmpdir, datafiles):
	287	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	288	+ workspace = os.path.join(cli.directory, 'workspace')
	289	+ element_name = 'autotools/amhello.bst'
	290	+
	291	+ # Ensure we're not using the shared artifact cache
	292	+ cli.configure({
	293	+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
	294	+ })
	295	+
	296	+ # First open the workspace
	297	+ res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
	298	+ res.assert_success()
	299	+
	300	+ # Check that by default the buildtree wasn't staged as not yet available in the cache
	301	+ assert not os.path.exists(os.path.join(workspace, 'src', 'hello'))
	302	+
	303	+ # Close the workspace, removing the dir
	304	+ res = cli.run(project=project, args=['workspace', 'close', '--remove-dir', element_name])
	305	+ res.assert_success()
	306	+
	307	+ # Build the element, so we have it cached along with the buildtreee
	308	+ res = cli.run(project=project, args=['build', element_name])
	309	+ res.assert_success()
	310	+
	311	+ # Open up the workspace, as the buildtree is cached by default it should open with the buildtree
	312	+ res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
	313	+ res.assert_success()
	314	+
	315	+ # Check that the buildtree was staged, by asserting output of the build exists in the dir
	316	+ assert os.path.exists(os.path.join(workspace, 'src', 'hello'))

tests/testutils/http_server.py

---

...	...	@@ -29,7 +29,7 @@ class RequestHandler(SimpleHTTPRequestHandler):
29	29	expected_password, directory = self.server.users[user]
30	30	if password == expected_password:
31	31	return directory
32		- except:
	32	+ except: # noqa
33	33	raise Unauthorized('unauthorized')
34	34	return None
35	35

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.