[Notes] [Git][BuildStream/buildstream][juerg/cas] 9 commits: loader: Add 'build-depends' and 'runtime-depends' fields to elements

[Jürg Billeter <gitlab mg gitlab com>]

Title: GitLab

Jürg Billeter pushed to branch juerg/cas at BuildStream / buildstream

Commits:

• 0c8d0b0b
  by Jonathan Maw at 2018-08-14T08:54:31Z

  loader: Add 'build-depends' and 'runtime-depends' fields to elements
  

• 95798475
  by Jonathan Maw at 2018-08-14T08:54:31Z

  tests: Add tests for loading builddeps and runtime deps
  

• 476fcfa6
  by Jonathan Maw at 2018-08-14T08:54:31Z

  versions: Bump format version to 14
  
  Format version raised because of a change to the core element format.
  

• c51bbff3
  by Jonathan Maw at 2018-08-14T08:54:31Z

  docs: Add documentation of build-depends and runtime-depends fields
  

• 45117239
  by Jonathan Maw at 2018-08-14T08:58:03Z

  Add NEWS
  

• 9d074d7f
  by Jonathan Maw at 2018-08-14T11:52:59Z

  Merge branch '463-make-dependency-type-default-to-build-2' into 'master'
  
  Resolve "Make dependency type default to build"
  
  Closes #463
  
  See merge request BuildStream/buildstream!633

• a10dfc5e
  by Jürg Billeter at 2018-08-14T15:08:14Z

  _artifactcache/casserver.py: Fix resource_name format for blobs
  
  Continue to accept requests from non-conforming BuildStream clients for
  the time being to ease migration.
  

• b7fa14cc
  by Jürg Billeter at 2018-08-14T15:08:14Z

  _artifactcache/cascache.py: Fix resource_name format for blobs
  
  This requires an updated server.
  
  Fixes #572.
  

• afac717d
  by Jürg Billeter at 2018-08-14T15:41:41Z

  _artifactcache/casserver.py: Improve ByteStream error handling
  
  Replace assertions with gRPC error responses.
  

15 changed files:

• NEWS
• buildstream/_artifactcache/cascache.py
• buildstream/_artifactcache/casserver.py
• buildstream/_loader/loadelement.py
• buildstream/_loader/types.py
• buildstream/_versions.py
• doc/source/format_declaring.rst
• tests/loader/dependencies.py
• + tests/loader/dependencies/elements/builddep-list.bst
• tests/loader/dependencies/elements/firstdep.bst
• + tests/loader/dependencies/elements/list-combine.bst
• + tests/loader/dependencies/elements/list-overlap.bst
• + tests/loader/dependencies/elements/runtimedep-list.bst
• + tests/loader/dependencies/elements/seconddep.bst
• + tests/loader/dependencies/elements/thirddep.bst

Changes:

NEWS

---

...	...	@@ -13,6 +13,10 @@ buildstream 1.3.1
13	13	Downgrading after using this feature may result in workspaces
14	14	not functioning correctly
15	15
	16	+ o Elements may now specify 'build-depends' and 'runtime-depends' fields
	17	+ to avoid having to specify the dependency type for every entry in
	18	+ 'depends'.
	19	+
16	20
17	21	=================
18	22	buildstream 1.1.5

buildstream/_artifactcache/cascache.py

---

...	...	@@ -24,6 +24,7 @@ import os
24	24	import signal
25	25	import stat
26	26	import tempfile
	27	+import uuid
27	28	from urllib.parse import urlparse
28	29
29	30	import grpc
...	...	@@ -309,8 +310,11 @@ class CASCache(ArtifactCache):
309	310	# Upload any blobs missing on the server
310	311	skipped_remote = False
311	312	for digest in missing_blobs.values():
	313	+ uuid_ = uuid.uuid4()
	314	+ resource_name = '/'.join(['uploads', str(uuid_), 'blobs',
	315	+ digest.hash, str(digest.size_bytes)])
	316	+
312	317	def request_stream():
313		- resource_name = os.path.join(digest.hash, str(digest.size_bytes))
314	318	with open(self.objpath(digest), 'rb') as f:
315	319	assert os.fstat(f.fileno()).st_size == digest.size_bytes
316	320	offset = 0
...	...	@@ -747,7 +751,7 @@ class CASCache(ArtifactCache):
747	751	yield from self._required_blobs(dirnode.digest)
748	752
749	753	def _fetch_blob(self, remote, digest, out):
750		- resource_name = os.path.join(digest.hash, str(digest.size_bytes))
	754	+ resource_name = '/'.join(['blobs', digest.hash, str(digest.size_bytes)])
751	755	request = bytestream_pb2.ReadRequest()
752	756	request.resource_name = resource_name
753	757	request.read_offset = 0

buildstream/_artifactcache/casserver.py

---

...	...	@@ -23,6 +23,7 @@ import os
23	23	import signal
24	24	import sys
25	25	import tempfile
	26	+import uuid
26	27
27	28	import click
28	29	import grpc
...	...	@@ -130,12 +131,21 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
130	131
131	132	def Read(self, request, context):
132	133	resource_name = request.resource_name
133		- client_digest = _digest_from_resource_name(resource_name)
134		- assert request.read_offset <= client_digest.size_bytes
	134	+ client_digest = _digest_from_download_resource_name(resource_name)
	135	+ if client_digest is None:
	136	+ context.set_code(grpc.StatusCode.NOT_FOUND)
	137	+ return
	138	+
	139	+ if request.read_offset > client_digest.size_bytes:
	140	+ context.set_code(grpc.StatusCode.OUT_OF_RANGE)
	141	+ return
135	142
136	143	try:
137	144	with open(self.cas.objpath(client_digest), 'rb') as f:
138		- assert os.fstat(f.fileno()).st_size == client_digest.size_bytes
	145	+ if os.fstat(f.fileno()).st_size != client_digest.size_bytes:
	146	+ context.set_code(grpc.StatusCode.NOT_FOUND)
	147	+ return
	148	+
139	149	if request.read_offset > 0:
140	150	f.seek(request.read_offset)
141	151
...	...	@@ -163,12 +173,18 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
163	173	resource_name = None
164	174	with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
165	175	for request in request_iterator:
166		- assert not finished
167		- assert request.write_offset == offset
	176	+ if finished or request.write_offset != offset:
	177	+ context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
	178	+ return response
	179	+
168	180	if resource_name is None:
169	181	# First request
170	182	resource_name = request.resource_name
171		- client_digest = _digest_from_resource_name(resource_name)
	183	+ client_digest = _digest_from_upload_resource_name(resource_name)
	184	+ if client_digest is None:
	185	+ context.set_code(grpc.StatusCode.NOT_FOUND)
	186	+ return response
	187	+
172	188	try:
173	189	_clean_up_cache(self.cas, client_digest.size_bytes)
174	190	except ArtifactTooLargeException as e:
...	...	@@ -177,14 +193,20 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
177	193	return response
178	194	elif request.resource_name:
179	195	# If it is set on subsequent calls, it **must** match the value of the first request.
180		- assert request.resource_name == resource_name
	196	+ if request.resource_name != resource_name:
	197	+ context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
	198	+ return response
181	199	out.write(request.data)
182	200	offset += len(request.data)
183	201	if request.finish_write:
184		- assert client_digest.size_bytes == offset
	202	+ if client_digest.size_bytes != offset:
	203	+ context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
	204	+ return response
185	205	out.flush()
186	206	digest = self.cas.add_object(path=out.name)
187		- assert digest.hash == client_digest.hash
	207	+ if digest.hash != client_digest.hash:
	208	+ context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
	209	+ return response
188	210	finished = True
189	211
190	212	assert finished
...	...	@@ -247,13 +269,48 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
247	269	return response
248	270
249	271
250		-def _digest_from_resource_name(resource_name):
	272	+def _digest_from_download_resource_name(resource_name):
	273	+ parts = resource_name.split('/')
	274	+
	275	+ # Accept requests from non-conforming BuildStream 1.1.x clients
	276	+ if len(parts) == 2:
	277	+ parts.insert(0, 'blobs')
	278	+
	279	+ if len(parts) != 3 or parts[0] != 'blobs':
	280	+ return None
	281	+
	282	+ try:
	283	+ digest = remote_execution_pb2.Digest()
	284	+ digest.hash = parts[1]
	285	+ digest.size_bytes = int(parts[2])
	286	+ return digest
	287	+ except ValueError:
	288	+ return None
	289	+
	290	+
	291	+def _digest_from_upload_resource_name(resource_name):
251	292	parts = resource_name.split('/')
252		- assert len(parts) == 2
253		- digest = remote_execution_pb2.Digest()
254		- digest.hash = parts[0]
255		- digest.size_bytes = int(parts[1])
256		- return digest
	293	+
	294	+ # Accept requests from non-conforming BuildStream 1.1.x clients
	295	+ if len(parts) == 2:
	296	+ parts.insert(0, 'uploads')
	297	+ parts.insert(1, str(uuid.uuid4()))
	298	+ parts.insert(2, 'blobs')
	299	+
	300	+ if len(parts) < 5 or parts[0] != 'uploads' or parts[2] != 'blobs':
	301	+ return None
	302	+
	303	+ try:
	304	+ uuid_ = uuid.UUID(hex=parts[1])
	305	+ if uuid_.version != 4:
	306	+ return None
	307	+
	308	+ digest = remote_execution_pb2.Digest()
	309	+ digest.hash = parts[3]
	310	+ digest.size_bytes = int(parts[4])
	311	+ return digest
	312	+ except ValueError:
	313	+ return None
257	314
258	315
259	316	def _has_object(cas, digest):

buildstream/_loader/loadelement.py

---

...	...	@@ -71,6 +71,7 @@ class LoadElement():
71	71	'kind', 'depends', 'sources', 'sandbox',
72	72	'variables', 'environment', 'environment-nocache',
73	73	'config', 'public', 'description',
	74	+ 'build-depends', 'runtime-depends',
74	75	])
75	76
76	77	# Extract the Dependencies
...	...	@@ -127,28 +128,46 @@ class LoadElement():
127	128	# Returns:
128	129	# (list): a list of Dependency objects
129	130	#
130		-def _extract_depends_from_node(node):
131		- depends = _yaml.node_get(node, list, Symbol.DEPENDS, default_value=[])
	131	+def _extract_depends_from_node(node, *, key=None):
	132	+ if key is None:
	133	+ build_depends = _extract_depends_from_node(node, key=Symbol.BUILD_DEPENDS)
	134	+ runtime_depends = _extract_depends_from_node(node, key=Symbol.RUNTIME_DEPENDS)
	135	+ depends = _extract_depends_from_node(node, key=Symbol.DEPENDS)
	136	+ return build_depends + runtime_depends + depends
	137	+ elif key == Symbol.BUILD_DEPENDS:
	138	+ default_dep_type = Symbol.BUILD
	139	+ elif key == Symbol.RUNTIME_DEPENDS:
	140	+ default_dep_type = Symbol.RUNTIME
	141	+ elif key == Symbol.DEPENDS:
	142	+ default_dep_type = None
	143	+ else:
	144	+ assert False, "Unexpected value of key '{}'".format(key)
	145	+
	146	+ depends = _yaml.node_get(node, list, key, default_value=[])
132	147	output_deps = []
133	148
134	149	for dep in depends:
135		- dep_provenance = _yaml.node_get_provenance(node, key=Symbol.DEPENDS, indices=[depends.index(dep)])
	150	+ dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[depends.index(dep)])
136	151
137	152	if isinstance(dep, str):
138		- dependency = Dependency(dep, provenance=dep_provenance)
	153	+ dependency = Dependency(dep, provenance=dep_provenance, dep_type=default_dep_type)
139	154
140	155	elif isinstance(dep, Mapping):
141		- _yaml.node_validate(dep, ['filename', 'type', 'junction'])
142		-
143		- # Make type optional, for this we set it to None
144		- dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
145		- if dep_type is None or dep_type == Symbol.ALL:
146		- dep_type = None
147		- elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
148		- provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
149		- raise LoadError(LoadErrorReason.INVALID_DATA,
150		- "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
151		- .format(provenance, dep_type))
	156	+ if default_dep_type:
	157	+ _yaml.node_validate(dep, ['filename', 'junction'])
	158	+ dep_type = default_dep_type
	159	+ else:
	160	+ _yaml.node_validate(dep, ['filename', 'type', 'junction'])
	161	+
	162	+ # Make type optional, for this we set it to None
	163	+ dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
	164	+ if dep_type is None or dep_type == Symbol.ALL:
	165	+ dep_type = None
	166	+ elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
	167	+ provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
	168	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	169	+ "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
	170	+ .format(provenance, dep_type))
152	171
153	172	filename = _yaml.node_get(dep, str, Symbol.FILENAME)
154	173	junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None)
...	...	@@ -159,13 +178,13 @@ def _extract_depends_from_node(node):
159	178
160	179	else:
161	180	index = depends.index(dep)
162		- p = _yaml.node_get_provenance(node, key=Symbol.DEPENDS, indices=[index])
	181	+ p = _yaml.node_get_provenance(node, key=key, indices=[index])
163	182	raise LoadError(LoadErrorReason.INVALID_DATA,
164	183	"{}: Dependency is not specified as a string or a dictionary".format(p))
165	184
166	185	output_deps.append(dependency)
167	186
168		- # Now delete "depends", we dont want it anymore
169		- del node[Symbol.DEPENDS]
	187	+ # Now delete the field, we dont want it anymore
	188	+ del node[key]
170	189
171	190	return output_deps

buildstream/_loader/types.py

---

...	...	@@ -26,6 +26,8 @@ class Symbol():
26	26	FILENAME = "filename"
27	27	KIND = "kind"
28	28	DEPENDS = "depends"
	29	+ BUILD_DEPENDS = "build-depends"
	30	+ RUNTIME_DEPENDS = "runtime-depends"
29	31	SOURCES = "sources"
30	32	CONFIG = "config"
31	33	VARIABLES = "variables"

buildstream/_versions.py

---

...	...	@@ -23,7 +23,7 @@
23	23	# This version is bumped whenever enhancements are made
24	24	# to the `project.conf` format or the core element format.
25	25	#
26		-BST_FORMAT_VERSION = 13
	26	+BST_FORMAT_VERSION = 14
27	27
28	28
29	29	# The base BuildStream artifact version

doc/source/format_declaring.rst

---

...	...	@@ -98,6 +98,68 @@ relative filename to the elements they depend on here.
98	98	See :ref:`format_dependencies` for more information on the dependency model.
99	99
100	100
	101	+.. _format_build_depends:
	102	+
	103	+Build-Depends
	104	+~~~~~~~~~~~~~
	105	+
	106	+.. code:: yaml
	107	+
	108	+ # Specify some build-dependencies
	109	+ build-depends:
	110	+ - element1.bst
	111	+ - element2.bst
	112	+
	113	+Build dependencies between elements can be specified with the ``build-depends`` attribute.
	114	+The above code snippet is equivalent to:
	115	+
	116	+.. code:: yaml
	117	+
	118	+ # Specify some build-dependencies
	119	+ depends:
	120	+ - filename: element1.bst
	121	+ type: build
	122	+ - filename: element2.bst
	123	+ type: build
	124	+
	125	+See :ref:`format_dependencies` for more information on the dependency model.
	126	+
	127	+.. note::
	128	+
	129	+ The ``build-depends`` configuration is available since :ref:`format version 14 <project_format_version>`
	130	+
	131	+
	132	+.. _format_runtime_depends:
	133	+
	134	+Runtime-Depends
	135	+~~~~~~~~~~~~~~~
	136	+
	137	+.. code:: yaml
	138	+
	139	+ # Specify some runtime-dependencies
	140	+ runtime-depends:
	141	+ - element1.bst
	142	+ - element2.bst
	143	+
	144	+Runtime dependencies between elements can be specified with the ``runtime-depends`` attribute.
	145	+The above code snippet is equivalent to:
	146	+
	147	+.. code:: yaml
	148	+
	149	+ # Specify some runtime-dependencies
	150	+ depends:
	151	+ - filename: element1.bst
	152	+ type: runtime
	153	+ - filename: element2.bst
	154	+ type: runtime
	155	+
	156	+See :ref:`format_dependencies` for more information on the dependency model.
	157	+
	158	+.. note::
	159	+
	160	+ The ``runtime-depends`` configuration is available since :ref:`format version 14 <project_format_version>`
	161	+
	162	+
101	163	.. _format_sources:
102	164
103	165	Sources
...	...	@@ -276,8 +338,8 @@ attributes are suitable.
276	338
277	339	.. note::
278	340
279		- Note the order in which element dependencies are declared in the ``depends``
280		- list is not meaningful.
	341	+ Note the order in which element dependencies are declared in the ``depends``,
	342	+ ``build-depends`` and ``runtime-depends`` lists are not meaningful.
281	343
282	344	Dependency dictionary:
283	345
...	...	@@ -299,6 +361,8 @@ Attributes:
299	361	* ``type``
300	362
301	363	This attribute is used to express the :ref:`dependency type <format_dependencies_types>`.
	364	+ This field is not permitted in :ref:`Build-Depends <format_build_depends>` or
	365	+ :ref:`Runtime-Depends <format_runtime_depends>`.
302	366
303	367	* ``junction``
304	368

tests/loader/dependencies.py

---

...	...	@@ -3,6 +3,7 @@ import pytest
3	3
4	4	from buildstream._exceptions import LoadError, LoadErrorReason
5	5	from buildstream._loader import Loader, MetaElement
	6	+from tests.testutils import cli
6	7	from . import make_loader
7	8
8	9	DATA_DIR = os.path.join(
...	...	@@ -27,7 +28,7 @@ def test_two_files(datafiles):
27	28	assert(len(element.dependencies) == 1)
28	29	firstdep = element.dependencies[0]
29	30	assert(isinstance(firstdep, MetaElement))
30		- assert(firstdep.kind == 'thefirstdep')
	31	+ assert(firstdep.kind == 'manual')
31	32
32	33
33	34	@pytest.mark.datafiles(DATA_DIR)
...	...	@@ -47,7 +48,7 @@ def test_shared_dependency(datafiles):
47	48	#
48	49	firstdep = element.dependencies[0]
49	50	assert(isinstance(firstdep, MetaElement))
50		- assert(firstdep.kind == 'thefirstdep')
	51	+ assert(firstdep.kind == 'manual')
51	52	assert(len(firstdep.dependencies) == 0)
52	53
53	54	# The second specified dependency is 'shareddep'
...	...	@@ -86,7 +87,7 @@ def test_dependency_dict(datafiles):
86	87	assert(len(element.dependencies) == 1)
87	88	firstdep = element.dependencies[0]
88	89	assert(isinstance(firstdep, MetaElement))
89		- assert(firstdep.kind == 'thefirstdep')
	90	+ assert(firstdep.kind == 'manual')
90	91
91	92
92	93	@pytest.mark.datafiles(DATA_DIR)
...	...	@@ -186,3 +187,49 @@ def test_all_dependency(datafiles):
186	187	assert(isinstance(firstdep, MetaElement))
187	188	firstbuilddep = element.build_dependencies[0]
188	189	assert(firstdep == firstbuilddep)
	190	+
	191	+
	192	+@pytest.mark.datafiles(DATA_DIR)
	193	+def test_list_build_dependency(cli, datafiles):
	194	+ project = str(datafiles)
	195	+
	196	+ # Check that the pipeline includes the build dependency
	197	+ deps = cli.get_pipeline(project, ['elements/builddep-list.bst'], scope="build")
	198	+ assert "elements/firstdep.bst" in deps
	199	+
	200	+
	201	+@pytest.mark.datafiles(DATA_DIR)
	202	+def test_list_runtime_dependency(cli, datafiles):
	203	+ project = str(datafiles)
	204	+
	205	+ # Check that the pipeline includes the runtime dependency
	206	+ deps = cli.get_pipeline(project, ['elements/runtimedep-list.bst'], scope="run")
	207	+ assert "elements/firstdep.bst" in deps
	208	+
	209	+
	210	+@pytest.mark.datafiles(DATA_DIR)
	211	+def test_list_dependencies_combined(cli, datafiles):
	212	+ project = str(datafiles)
	213	+
	214	+ # Check that runtime deps get combined
	215	+ rundeps = cli.get_pipeline(project, ['elements/list-combine.bst'], scope="run")
	216	+ assert "elements/firstdep.bst" not in rundeps
	217	+ assert "elements/seconddep.bst" in rundeps
	218	+ assert "elements/thirddep.bst" in rundeps
	219	+
	220	+ # Check that build deps get combined
	221	+ builddeps = cli.get_pipeline(project, ['elements/list-combine.bst'], scope="build")
	222	+ assert "elements/firstdep.bst" in builddeps
	223	+ assert "elements/seconddep.bst" not in builddeps
	224	+ assert "elements/thirddep.bst" in builddeps
	225	+
	226	+
	227	+@pytest.mark.datafiles(DATA_DIR)
	228	+def test_list_overlap(cli, datafiles):
	229	+ project = str(datafiles)
	230	+
	231	+ # Check that dependencies get merged
	232	+ rundeps = cli.get_pipeline(project, ['elements/list-overlap.bst'], scope="run")
	233	+ assert "elements/firstdep.bst" in rundeps
	234	+ builddeps = cli.get_pipeline(project, ['elements/list-overlap.bst'], scope="build")
	235	+ assert "elements/firstdep.bst" in builddeps

tests/loader/dependencies/elements/builddep-list.bst

---

	1	+kind: stack
	2	+description: This element has a build-only dependency specified via build-depends
	3	+build-depends:
	4	+ - elements/firstdep.bst

tests/loader/dependencies/elements/firstdep.bst

---

1		-kind: thefirstdep
	1	+kind: manual
2	2	description: This is the first dependency

tests/loader/dependencies/elements/list-combine.bst

---

	1	+kind: stack
	2	+description: This element depends on three elements in different ways
	3	+build-depends:
	4	+- elements/firstdep.bst
	5	+runtime-depends:
	6	+- elements/seconddep.bst
	7	+depends:
	8	+- elements/thirddep.bst

tests/loader/dependencies/elements/list-overlap.bst

---

	1	+kind: stack
	2	+description: This element depends on two elements in different ways
	3	+build-depends:
	4	+- elements/firstdep.bst
	5	+depends:
	6	+- filename: elements/firstdep.bst
	7	+ type: runtime

tests/loader/dependencies/elements/runtimedep-list.bst

---

	1	+kind: stack
	2	+description: This element has a runtime-only dependency
	3	+runtime-depends:
	4	+ - elements/firstdep.bst

tests/loader/dependencies/elements/seconddep.bst

---

	1	+kind: manual
	2	+description: This is the second dependency

tests/loader/dependencies/elements/thirddep.bst

---

	1	+kind: manual
	2	+description: This is the third dependency

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.