[Notes] [Git][BuildStream/buildstream][richardmaw/artifact-log] 7 commits: CasBasedDirectory: Change constructor to take a CASCache instead of a Context

[richardmaw-codethink <gitlab mg gitlab com>]

Title: GitLab

richardmaw-codethink pushed to branch richardmaw/artifact-log at BuildStream / buildstream

Commits:

• ee7354f6
  by Richard Maw at 2018-11-14T18:01:32Z

  CasBasedDirectory: Change constructor to take a CASCache instead of a Context
  
  The Context was only used to obtain a reference to the CASCache
  and set the unused cas_directory field.
  

• 8fc6cd22
  by Richard Maw at 2018-11-14T18:01:32Z

  _artifactcache: Add get_virtual_directory method
  
  If all you want is a subset of an artifact's output
  using a cached extracted tree is unnecessary work.
  
  By exposing the virtual CasBasedDirectory we can use `descend` to select a subtree
  and `export_files` to read the contents.
  

• eb23a8cc
  by Richard Maw at 2018-11-14T18:01:32Z

  cli: Add support for auto-completing artifact ref names
  

• 7d1c643e
  by Richard Maw at 2018-11-14T18:01:32Z

  cli: Add artifact command group
  

• e3c80a3b
  by Richard Maw at 2018-11-14T18:01:32Z

  cli: Add artifact log command
  

• 4c90e066
  by Richard Maw at 2018-11-14T18:01:32Z

  tests: Add tests for artifact group commands
  

• 886303c8
  by Richard Maw at 2018-11-14T18:01:32Z

  NEWS: Mention `bst artifact log`
  
  Closes https://gitlab.com/BuildStream/buildstream/issues/234
  

12 changed files:

• NEWS
• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_frontend/cli.py
• buildstream/sandbox/_sandboxremote.py
• buildstream/sandbox/sandbox.py
• buildstream/storage/_casbaseddirectory.py
• tests/artifactcache/push.py
• tests/completions/completions.py
• + tests/integration/artifact.py
• tests/sandboxes/storage-tests.py
• tests/storage/virtual_directory_import.py

Changes:

NEWS

---

...	...	@@ -2,6 +2,8 @@
2	2	buildstream 1.3.1
3	3	=================
4	4
	5	+ o Added `bst artifact log` subcommand for viewing build logs.
	6	+
5	7	o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
6	8	environment variables. There is already a 'make' element with the same
7	9	variables. Note that this is a breaking change, it will require users to

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -547,6 +547,27 @@ class ArtifactCache():
547	547
548	548	return self.cas.extract(ref, path)
549	549
	550	+ # get_virtual_directory():
	551	+ #
	552	+ # Get the contents of the artifact as a buildstream.storage.Directory.
	553	+ #
	554	+ # The returned Directory may become invalid if remove or prune are called.
	555	+ #
	556	+ # Assumes artifact has previously been fetched or committed.
	557	+ #
	558	+ # Args:
	559	+ # element (Element): The Element to get the Directory of
	560	+ # key (str): The cache key to use
	561	+ #
	562	+ # Raises:
	563	+ # ArtifactError: If the artifact did not exist.
	564	+ #
	565	+ # Returns: virtual Directory
	566	+ #
	567	+ def get_virtual_directory(self, element, key):
	568	+ ref = self.get_artifact_fullname(element, key)
	569	+ return self.cas.get_virtual_directory(ref)
	570	+
550	571	# commit():
551	572	#
552	573	# Commit built artifact to cache.

buildstream/_artifactcache/cascache.py

---

...	...	@@ -36,6 +36,7 @@ from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
36	36
37	37	from .. import utils
38	38	from .._exceptions import CASError
	39	+from ..storage._casbaseddirectory import CasBasedDirectory
39	40
40	41
41	42	# The default limit for gRPC messages is 4 MiB.
...	...	@@ -122,6 +123,23 @@ class CASCache():
122	123
123	124	return dest
124	125
	126	+ # get_virtual_directory():
	127	+ #
	128	+ # Get the contents of the ref as a buildstream.storage.Directory.
	129	+ # The returned Directory may become invalid if remove or prune are called.
	130	+ #
	131	+ # Args:
	132	+ # ref (str): The ref whose directory to extract
	133	+ #
	134	+ # Raises:
	135	+ # CASError: If the ref did not exist.
	136	+ #
	137	+ # Returns: virtual Directory
	138	+ #
	139	+ def get_virtual_directory(self, ref):
	140	+ tree = self.resolve_ref(ref, update_mtime=True)
	141	+ return CasBasedDirectory(self, ref=tree)
	142	+
125	143	# commit():
126	144	#
127	145	# Commit directory to cache.

buildstream/_frontend/cli.py

---

1	1	import os
2	2	import sys
	3	+from contextlib import ExitStack
	4	+from tempfile import TemporaryDirectory
3	5
4	6	import click
5	7	from .. import _yaml
...	...	@@ -112,6 +114,18 @@ def complete_target(args, incomplete):
112	114	return complete_path("File", incomplete, base_directory=base_directory)
113	115
114	116
	117	+def complete_artifact(args, incomplete):
	118	+ from .._context import Context
	119	+ ctx = Context()
	120	+
	121	+ config = None
	122	+ for i, arg in enumerate(args):
	123	+ if arg in ('-c', '--config'):
	124	+ config = args[i + 1]
	125	+ ctx.load(config)
	126	+ return [ref for ref in ctx.artifactcache.cas.list_refs() if ref.startswith(incomplete)]
	127	+
	128	+
115	129	def override_completions(cmd, cmd_param, args, incomplete):
116	130	"""
117	131	:param cmd_param: command definition
...	...	@@ -126,13 +140,15 @@ def override_completions(cmd, cmd_param, args, incomplete):
126	140	# We can't easily extend click's data structures without
127	141	# modifying click itself, so just do some weak special casing
128	142	# right here and select which parameters we want to handle specially.
129		- if isinstance(cmd_param.type, click.Path) and \
130		- (cmd_param.name == 'elements' or
131		- cmd_param.name == 'element' or
132		- cmd_param.name == 'except_' or
133		- cmd_param.opts == ['--track'] or
134		- cmd_param.opts == ['--track-except']):
135		- return complete_target(args, incomplete)
	143	+ if isinstance(cmd_param.type, click.Path):
	144	+ if (cmd_param.name == 'elements' or
	145	+ cmd_param.name == 'element' or
	146	+ cmd_param.name == 'except_' or
	147	+ cmd_param.opts == ['--track'] or
	148	+ cmd_param.opts == ['--track-except']):
	149	+ return complete_target(args, incomplete)
	150	+ if cmd_param.name == 'artifacts':
	151	+ return complete_artifact(args, incomplete)
136	152
137	153	raise CompleteUnhandled()
138	154
...	...	@@ -829,3 +845,64 @@ def source_bundle(app, element, force, directory,
829	845	force=force,
830	846	compression=compression,
831	847	except_targets=except_)
	848	+
	849	+
	850	+#############################################################
	851	+# Artifact Command #
	852	+#############################################################
	853	+@cli.group(short_help="Manipulate cached Artifacts")
	854	+def artifact():
	855	+ """Manipulate cached Artifacts"""
	856	+ pass
	857	+
	858	+
	859	+################################################################
	860	+# Artifact Log Command #
	861	+################################################################
	862	+@artifact.command(name='log', short_help="Show logs of an artifact")
	863	+@click.option('-e', '--element', 'elements', help="Show logs for artifacts of this element",
	864	+ type=click.Path(readable=False), multiple=True, required=False)
	865	+@click.argument('artifacts', type=click.Path(), nargs=-1)
	866	+@click.pass_obj
	867	+def artifact_log(app, elements, artifacts):
	868	+ """Show logs of all artifacts"""
	869	+ from .._exceptions import CASError
	870	+ from .._message import MessageType
	871	+ from .._pipeline import PipelineSelection
	872	+
	873	+ with ExitStack() as stack:
	874	+ stack.enter_context(app.initialized())
	875	+ cache = app.context.artifactcache
	876	+
	877	+ vdirs = []
	878	+ extractdirs = []
	879	+ if artifacts:
	880	+ for ref in artifacts:
	881	+ try:
	882	+ vdir = cache.cas.get_virtual_directory(ref)
	883	+ vdirs.append(vdir)
	884	+ except CASError as e:
	885	+ app._message(MessageType.WARN, "Artifact {} is not cached".format(ref), detail=str(e))
	886	+ continue
	887	+ if elements:
	888	+ elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
	889	+ for element in elements:
	890	+ if not element._cached():
	891	+ app._message(MessageType.WARN, "Element {} is not cached".format(element))
	892	+ continue
	893	+ vdir = cache.get_virtual_directory(element, element._get_cache_key())
	894	+ vdirs.append(vdir)
	895	+
	896	+ for vdir in vdirs:
	897	+ # NOTE: If this feels unresponsive, this should provide some progress information.
	898	+ logsdir = vdir.descend(["logs"])
	899	+ td = stack.enter_context(TemporaryDirectory())
	900	+ logsdir.export_files(td, can_link=True)
	901	+ extractdirs.append(td)
	902	+
	903	+ for extractdir in extractdirs:
	904	+ for log in (os.path.join(extractdir, log) for log in os.listdir(extractdir)):
	905	+ # NOTE: Should click gain the ability to pass files to the pager this can be optimised.
	906	+ with open(log) as f:
	907	+ data = f.read()
	908	+ click.echo_via_pager(data)

buildstream/sandbox/_sandboxremote.py

---

...	...	@@ -182,7 +182,7 @@ class SandboxRemote(Sandbox):
182	182	# to replace the sandbox's virtual directory with that. Creating a new virtual directory object
183	183	# from another hash will be interesting, though...
184	184
185		- new_dir = CasBasedDirectory(self._get_context(), ref=dir_digest)
	185	+ new_dir = CasBasedDirectory(self._get_context().artifactcache.cas, ref=dir_digest)
186	186	self._set_virtual_directory(new_dir)
187	187
188	188	def run(self, command, flags, *, cwd=None, env=None):
...	...	@@ -191,7 +191,7 @@ class SandboxRemote(Sandbox):
191	191
192	192	if isinstance(upload_vdir, FileBasedDirectory):
193	193	# Make a new temporary directory to put source in
194		- upload_vdir = CasBasedDirectory(self._get_context(), ref=None)
	194	+ upload_vdir = CasBasedDirectory(self._get_context().artifactcache.cas, ref=None)
195	195	upload_vdir.import_files(self.get_virtual_directory()._get_underlying_directory())
196	196
197	197	upload_vdir.recalculate_hash()

buildstream/sandbox/sandbox.py

---

...	...	@@ -156,7 +156,7 @@ class Sandbox():
156	156	"""
157	157	if self._vdir is None or self._never_cache_vdirs:
158	158	if 'BST_CAS_DIRECTORIES' in os.environ:
159		- self._vdir = CasBasedDirectory(self.__context, ref=None)
	159	+ self._vdir = CasBasedDirectory(self.__context.artifactcache.cas, ref=None)
160	160	else:
161	161	self._vdir = FileBasedDirectory(self._root)
162	162	return self._vdir

buildstream/storage/_casbaseddirectory.py

---

...	...	@@ -249,13 +249,11 @@ class CasBasedDirectory(Directory):
249	249	_pb2_path_sep = "/"
250	250	_pb2_absolute_path_prefix = "/"
251	251
252		- def __init__(self, context, ref=None, parent=None, common_name="untitled", filename=None):
253		- self.context = context
254		- self.cas_directory = os.path.join(context.artifactdir, 'cas')
	252	+ def __init__(self, cas_cache, ref=None, parent=None, common_name="untitled", filename=None):
255	253	self.filename = filename
256	254	self.common_name = common_name
257	255	self.pb2_directory = remote_execution_pb2.Directory()
258		- self.cas_cache = context.artifactcache.cas
	256	+ self.cas_cache = cas_cache
259	257	if ref:
260	258	with open(self.cas_cache.objpath(ref), 'rb') as f:
261	259	self.pb2_directory.ParseFromString(f.read())
...	...	@@ -270,7 +268,7 @@ class CasBasedDirectory(Directory):
270	268	if self._directory_read:
271	269	return
272	270	for entry in self.pb2_directory.directories:
273		- buildStreamDirectory = CasBasedDirectory(self.context, ref=entry.digest,
	271	+ buildStreamDirectory = CasBasedDirectory(self.cas_cache, ref=entry.digest,
274	272	parent=self, filename=entry.name)
275	273	self.index[entry.name] = IndexEntry(entry, buildstream_object=buildStreamDirectory)
276	274	for entry in self.pb2_directory.files:
...	...	@@ -333,7 +331,7 @@ class CasBasedDirectory(Directory):
333	331	.format(name, str(self), type(newdir)))
334	332	dirnode = self._find_pb2_entry(name)
335	333	else:
336		- newdir = CasBasedDirectory(self.context, parent=self, filename=name)
	334	+ newdir = CasBasedDirectory(self.cas_cache, parent=self, filename=name)
337	335	dirnode = self.pb2_directory.directories.add()
338	336
339	337	dirnode.name = name

tests/artifactcache/push.py

---

...	...	@@ -225,7 +225,7 @@ def _test_push_directory(user_config_file, project_dir, artifact_dir, artifact_d
225	225
226	226	if cas.has_push_remotes():
227	227	# Create a CasBasedDirectory from local CAS cache content
228		- directory = CasBasedDirectory(context, ref=artifact_digest)
	228	+ directory = CasBasedDirectory(context.artifactcache.cas, ref=artifact_digest)
229	229
230	230	# Push the CasBasedDirectory object
231	231	cas.push_directory(project, directory)

tests/completions/completions.py

---

...	...	@@ -6,6 +6,7 @@ from tests.testutils import cli
6	6	DATA_DIR = os.path.dirname(os.path.realpath(__file__))
7	7
8	8	MAIN_COMMANDS = [
	9	+ 'artifact ',
9	10	'build ',
10	11	'checkout ',
11	12	'fetch ',

tests/integration/artifact.py

---

	1	+#
	2	+# Copyright (C) 2018 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+#
	18	+# Authors: Richard Maw <richard maw codethink co uk>
	19	+#
	20	+
	21	+import os
	22	+import pytest
	23	+
	24	+from tests.testutils import cli_integration as cli
	25	+
	26	+
	27	+pytestmark = pytest.mark.integration
	28	+
	29	+
	30	+# Project directory
	31	+DATA_DIR = os.path.join(
	32	+ os.path.dirname(os.path.realpath(__file__)),
	33	+ "project",
	34	+)
	35	+
	36	+
	37	+@pytest.mark.integration
	38	+@pytest.mark.datafiles(DATA_DIR)
	39	+def test_artifact_log(cli, tmpdir, datafiles):
	40	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	41	+
	42	+ # Get the cache key of our test element
	43	+ result = cli.run(project=project, silent=True, args=[
	44	+ '--no-colors',
	45	+ 'show', '--deps', 'none', '--format', '%{full-key}',
	46	+ 'base.bst'
	47	+ ])
	48	+ key = result.output.strip()
	49	+
	50	+ # Ensure we have an artifact to read
	51	+ result = cli.run(project=project, args=['build', 'base.bst'])
	52	+ assert result.exit_code == 0
	53	+
	54	+ # Read the log via the element name
	55	+ result = cli.run(project=project, args=['artifact', 'log', '-e', 'base.bst'])
	56	+ assert result.exit_code == 0
	57	+ log = result.output
	58	+
	59	+ # Read the log via the key
	60	+ result = cli.run(project=project, args=['artifact', 'log', 'test/base/' + key])
	61	+ assert result.exit_code == 0
	62	+ assert log == result.output

tests/sandboxes/storage-tests.py

---

...	...	@@ -3,7 +3,7 @@ import pytest
3	3
4	4	from buildstream._exceptions import ErrorDomain
5	5
6		-from buildstream._context import Context
	6	+from buildstream._artifactcache.cascache import CASCache
7	7	from buildstream.storage._casbaseddirectory import CasBasedDirectory
8	8	from buildstream.storage._filebaseddirectory import FileBasedDirectory
9	9
...	...	@@ -17,9 +17,8 @@ def setup_backend(backend_class, tmpdir):
17	17	if backend_class == FileBasedDirectory:
18	18	return backend_class(os.path.join(tmpdir, "vdir"))
19	19	else:
20		- context = Context()
21		- context.artifactdir = os.path.join(tmpdir, "cas")
22		- return backend_class(context)
	20	+ cas_cache = CASCache(tmpdir)
	21	+ return backend_class(cas_cache)
23	22
24	23
25	24	@pytest.mark.parametrize("backend", [

tests/storage/virtual_directory_import.py

---

...	...	@@ -15,18 +15,6 @@ from buildstream import utils
15	15	# These are comparitive tests that check that FileBasedDirectory and
16	16	# CasBasedDirectory act identically.
17	17
18		-
19		-class FakeArtifactCache():
20		- def __init__(self):
21		- self.cas = None
22		-
23		-
24		-class FakeContext():
25		- def __init__(self):
26		- self.artifactdir = ''
27		- self.artifactcache = FakeArtifactCache()
28		-
29		-
30	18	# This is a set of example file system contents. It's a set of trees
31	19	# which are either expected to be problematic or were found to be
32	20	# problematic during random testing.
...	...	@@ -120,8 +108,8 @@ def file_contents_are(path, contents):
120	108	return file_contents(path) == contents
121	109
122	110
123		-def create_new_casdir(root_number, fake_context, tmpdir):
124		- d = CasBasedDirectory(fake_context)
	111	+def create_new_casdir(root_number, cas_cache, tmpdir):
	112	+ d = CasBasedDirectory(cas_cache)
125	113	d.import_files(os.path.join(tmpdir, "content", "root{}".format(root_number)))
126	114	assert d.ref.hash != empty_hash_ref
127	115	return d
...	...	@@ -175,20 +163,19 @@ def directory_not_empty(path):
175	163
176	164
177	165	def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False):
178		- fake_context = FakeContext()
179		- fake_context.artifactcache.cas = CASCache(tmpdir)
	166	+ cas_cache = CASCache(tmpdir)
180	167	# Create some fake content
181	168	generator_function(original, tmpdir)
182	169	if original != overlay:
183	170	generator_function(overlay, tmpdir)
184	171
185		- d = create_new_casdir(original, fake_context, tmpdir)
	172	+ d = create_new_casdir(original, cas_cache, tmpdir)
186	173
187		- duplicate_cas = create_new_casdir(original, fake_context, tmpdir)
	174	+ duplicate_cas = create_new_casdir(original, cas_cache, tmpdir)
188	175
189	176	assert duplicate_cas.ref.hash == d.ref.hash
190	177
191		- d2 = create_new_casdir(overlay, fake_context, tmpdir)
	178	+ d2 = create_new_casdir(overlay, cas_cache, tmpdir)
192	179	d.import_files(d2)
193	180	export_dir = os.path.join(tmpdir, "output-{}-{}".format(original, overlay))
194	181	roundtrip_dir = os.path.join(tmpdir, "roundtrip-{}-{}".format(original, overlay))
...	...	@@ -247,15 +234,14 @@ def test_random_cas_import(cli, tmpdir, original):
247	234
248	235
249	236	def _listing_test(tmpdir, root, generator_function):
250		- fake_context = FakeContext()
251		- fake_context.artifactcache.cas = CASCache(tmpdir)
	237	+ cas_cache = CASCache(tmpdir)
252	238	# Create some fake content
253	239	generator_function(root, tmpdir)
254	240
255	241	d = create_new_filedir(root, tmpdir)
256	242	filelist = list(d.list_relative_paths())
257	243
258		- d2 = create_new_casdir(root, fake_context, tmpdir)
	244	+ d2 = create_new_casdir(root, cas_cache, tmpdir)
259	245	filelist2 = list(d2.list_relative_paths())
260	246
261	247	assert filelist == filelist2

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.