[Notes] [Git][BuildStream/buildstream][jmac/remote_execution_split] 7 commits: artifactcache: Move ArtifactCacheSpec code into CASRemoteSpec.

[Jim MacArthur <gitlab mg gitlab com>]

Title: GitLab

Jim MacArthur pushed to branch jmac/remote_execution_split at BuildStream / buildstream

Commits:

• b42defc0
  by Jim MacArthur at 2018-11-22T18:56:52Z

  artifactcache: Move ArtifactCacheSpec code into CASRemoteSpec.
  
  There is nothing in ArtifactCacheSpec that's actually specific to
  artifacts, so I've made it a CAS class so we can use the same
  spec for remote execution.
  

• 1abea743
  by Jim MacArthur at 2018-11-22T18:56:52Z

  artifactcache.py, _context.py: Move CASCache object into context
  
  Since the artifact cache and remote execution share the same
  local CAS store, they should share the same CASCache object. Moving
  this into context allows us to do this.
  

• 094d3185
  by Jim MacArthur at 2018-11-22T19:15:08Z

  _sandboxremote: Remove references to artifact cache.
  
  Uses the new config options introduced earlier to specify remote
  exec storage and execution service.
  

• 7208078a
  by Jim MacArthur at 2018-11-22T19:15:17Z

  Documentation: Update to show new config options
  

• b13754f9
  by Jim MacArthur at 2018-11-22T19:15:17Z

  tests/sandboxes/remote-exec-config.py: New file.
  
  Tests that we get a sensible error message when we specify incomplete
  or wrong information in the configuration for remote execution.
  

• 835f403a
  by Jim MacArthur at 2018-11-22T19:15:17Z

  artifactcache.py: Remove unused verify_digest_pushed
  

• 494fcf7b
  by Jim MacArthur at 2018-11-22T19:15:17Z

  _sandboxremote.py: Remove print statements and correct some mistakes
  

9 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_context.py
• buildstream/_project.py
• buildstream/data/projectconfig.yaml
• buildstream/element.py
• buildstream/sandbox/_sandboxremote.py
• doc/source/format_project.rst
• + tests/sandboxes/remote-exec-config.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -21,7 +21,6 @@ import multiprocessing
21	21	import os
22	22	import signal
23	23	import string
24		-from collections import namedtuple
25	24	from collections.abc import Mapping
26	25
27	26	from ..types import _KeyStrength
...	...	@@ -31,7 +30,7 @@ from .. import _signals
31	30	from .. import utils
32	31	from .. import _yaml
33	32
34		-from .cascache import CASCache, CASRemote
	33	+from .cascache import CASRemote, CASRemoteSpec
35	34
36	35
37	36	CACHE_SIZE_FILE = "cache_size"
...	...	@@ -45,48 +44,8 @@ CACHE_SIZE_FILE = "cache_size"
45	44	# push (bool): Whether we should attempt to push artifacts to this cache,
46	45	# in addition to pulling from it.
47	46	#
48		-class ArtifactCacheSpec(namedtuple('ArtifactCacheSpec', 'url push server_cert client_key client_cert')):
49		-
50		- # _new_from_config_node
51		- #
52		- # Creates an ArtifactCacheSpec() from a YAML loaded node
53		- #
54		- @staticmethod
55		- def _new_from_config_node(spec_node, basedir=None):
56		- _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert'])
57		- url = _yaml.node_get(spec_node, str, 'url')
58		- push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
59		- if not url:
60		- provenance = _yaml.node_get_provenance(spec_node, 'url')
61		- raise LoadError(LoadErrorReason.INVALID_DATA,
62		- "{}: empty artifact cache URL".format(provenance))
63		-
64		- server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None)
65		- if server_cert and basedir:
66		- server_cert = os.path.join(basedir, server_cert)
67		-
68		- client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None)
69		- if client_key and basedir:
70		- client_key = os.path.join(basedir, client_key)
71		-
72		- client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None)
73		- if client_cert and basedir:
74		- client_cert = os.path.join(basedir, client_cert)
75		-
76		- if client_key and not client_cert:
77		- provenance = _yaml.node_get_provenance(spec_node, 'client-key')
78		- raise LoadError(LoadErrorReason.INVALID_DATA,
79		- "{}: 'client-key' was specified without 'client-cert'".format(provenance))
80		-
81		- if client_cert and not client_key:
82		- provenance = _yaml.node_get_provenance(spec_node, 'client-cert')
83		- raise LoadError(LoadErrorReason.INVALID_DATA,
84		- "{}: 'client-cert' was specified without 'client-key'".format(provenance))
85		-
86		- return ArtifactCacheSpec(url, push, server_cert, client_key, client_cert)
87		-
88		-
89		-ArtifactCacheSpec.__new__.__defaults__ = (None, None, None)
	47	+class ArtifactCacheSpec(CASRemoteSpec):
	48	+ pass
90	49
91	50
92	51	# An ArtifactCache manages artifacts.
...	...	@@ -99,7 +58,7 @@ class ArtifactCache():
99	58	self.context = context
100	59	self.extractdir = os.path.join(context.artifactdir, 'extract')
101	60
102		- self.cas = CASCache(context.artifactdir)
	61	+ self.cas = context.get_cascache()
103	62
104	63	self.global_remote_specs = []
105	64	self.project_remote_specs = {}
...	...	@@ -792,34 +751,6 @@ class ArtifactCache():
792	751
793	752	return message_digest
794	753
795		- # verify_digest_pushed():
796		- #
797		- # Check whether the object is already on the server in which case
798		- # there is no need to upload it.
799		- #
800		- # Args:
801		- # project (Project): The current project
802		- # digest (Digest): The object digest.
803		- #
804		- def verify_digest_pushed(self, project, digest):
805		-
806		- if self._has_push_remotes:
807		- push_remotes = [r for r in self._remotes[project] if r.spec.push]
808		- else:
809		- push_remotes = []
810		-
811		- if not push_remotes:
812		- raise ArtifactError("verify_digest_pushed was called, but no remote artifact " +
813		- "servers are configured as push remotes.")
814		-
815		- pushed = False
816		-
817		- for remote in push_remotes:
818		- if self.cas.verify_digest_on_remote(remote, digest):
819		- pushed = True
820		-
821		- return pushed
822		-
823	754	# link_key():
824	755	#
825	756	# Add a key for an existing artifact.

buildstream/_artifactcache/cascache.py

---

...	...	@@ -17,6 +17,7 @@
17	17	# Authors:
18	18	# Jürg Billeter <juerg billeter codethink co uk>
19	19
	20	+from collections import namedtuple
20	21	import hashlib
21	22	import itertools
22	23	import io
...	...	@@ -34,7 +35,8 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remo
34	35	from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
35	36
36	37	from .. import utils
37		-from .._exceptions import CASError
	38	+from .._exceptions import CASError, LoadError, LoadErrorReason
	39	+from .. import _yaml
38	40
39	41
40	42	# The default limit for gRPC messages is 4 MiB.
...	...	@@ -42,6 +44,50 @@ from .._exceptions import CASError
42	44	_MAX_PAYLOAD_BYTES = 1024 * 1024
43	45
44	46
	47	+class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key client_cert')):
	48	+
	49	+ # _new_from_config_node
	50	+ #
	51	+ # Creates an CASRemoteSpec() from a YAML loaded node
	52	+ #
	53	+ @staticmethod
	54	+ def _new_from_config_node(spec_node, basedir=None):
	55	+ _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert'])
	56	+ url = _yaml.node_get(spec_node, str, 'url')
	57	+ push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
	58	+ if not url:
	59	+ provenance = _yaml.node_get_provenance(spec_node, 'url')
	60	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	61	+ "{}: empty artifact cache URL".format(provenance))
	62	+
	63	+ server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None)
	64	+ if server_cert and basedir:
	65	+ server_cert = os.path.join(basedir, server_cert)
	66	+
	67	+ client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None)
	68	+ if client_key and basedir:
	69	+ client_key = os.path.join(basedir, client_key)
	70	+
	71	+ client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None)
	72	+ if client_cert and basedir:
	73	+ client_cert = os.path.join(basedir, client_cert)
	74	+
	75	+ if client_key and not client_cert:
	76	+ provenance = _yaml.node_get_provenance(spec_node, 'client-key')
	77	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	78	+ "{}: 'client-key' was specified without 'client-cert'".format(provenance))
	79	+
	80	+ if client_cert and not client_key:
	81	+ provenance = _yaml.node_get_provenance(spec_node, 'client-cert')
	82	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	83	+ "{}: 'client-cert' was specified without 'client-key'".format(provenance))
	84	+
	85	+ return CASRemoteSpec(url, push, server_cert, client_key, client_cert)
	86	+
	87	+
	88	+CASRemoteSpec.__new__.__defaults__ = (None, None, None)
	89	+
	90	+
45	91	# A CASCache manages a CAS repository as specified in the Remote Execution API.
46	92	#
47	93	# Args:

buildstream/_context.py

---

...	...	@@ -31,6 +31,7 @@ from ._exceptions import LoadError, LoadErrorReason, BstError
31	31	from ._message import Message, MessageType
32	32	from ._profile import Topics, profile_start, profile_end
33	33	from ._artifactcache import ArtifactCache
	34	+from ._artifactcache.cascache import CASCache
34	35	from ._workspaces import Workspaces
35	36	from .plugin import _plugin_lookup
36	37
...	...	@@ -141,6 +142,7 @@ class Context():
141	142	self._workspaces = None
142	143	self._log_handle = None
143	144	self._log_filename = None
	145	+ self._cascache = None
144	146
145	147	# load()
146	148	#
...	...	@@ -620,6 +622,11 @@ class Context():
620	622	if not os.environ.get('XDG_DATA_HOME'):
621	623	os.environ['XDG_DATA_HOME'] = os.path.expanduser('~/.local/share')
622	624
	625	+ def get_cascache(self):
	626	+ if self._cascache is None:
	627	+ self._cascache = CASCache(self.artifactdir)
	628	+ return self._cascache
	629	+
623	630
624	631	# _node_get_option_str()
625	632	#

buildstream/_project.py

---

...	...	@@ -30,6 +30,7 @@ from ._profile import Topics, profile_start, profile_end
30	30	from ._exceptions import LoadError, LoadErrorReason
31	31	from ._options import OptionPool
32	32	from ._artifactcache import ArtifactCache
	33	+from .sandbox import SandboxRemote
33	34	from ._elementfactory import ElementFactory
34	35	from ._sourcefactory import SourceFactory
35	36	from .plugin import CoreWarnings
...	...	@@ -130,7 +131,7 @@ class Project():
130	131	self._shell_host_files = [] # A list of HostMount objects
131	132
132	133	self.artifact_cache_specs = None
133		- self.remote_execution_url = None
	134	+ self.remote_execution_specs = None
134	135	self._sandbox = None
135	136	self._splits = None
136	137
...	...	@@ -493,9 +494,7 @@ class Project():
493	494	self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
494	495
495	496	# Load remote-execution configuration for this project
496		- remote_execution = _yaml.node_get(config, Mapping, 'remote-execution')
497		- _yaml.node_validate(remote_execution, ['url'])
498		- self.remote_execution_url = _yaml.node_get(remote_execution, str, 'url')
	497	+ self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
499	498
500	499	# Load sandbox environment variables
501	500	self.base_environment = _yaml.node_get(config, Mapping, 'environment')

buildstream/data/projectconfig.yaml

---

...	...	@@ -197,6 +197,3 @@ shell:
197	197	# Command to run when `bst shell` does not provide a command
198	198	#
199	199	command: [ 'sh', '-i' ]
200		-
201		-remote-execution:
202		- url: ""
		\ No newline at end of file

buildstream/element.py

---

...	...	@@ -250,9 +250,9 @@ class Element(Plugin):
250	250
251	251	# Extract remote execution URL
252	252	if not self.__is_junction:
253		- self.__remote_execution_url = project.remote_execution_url
	253	+ self.__remote_execution_specs = project.remote_execution_specs
254	254	else:
255		- self.__remote_execution_url = None
	255	+ self.__remote_execution_specs = None
256	256
257	257	# Extract Sandbox config
258	258	self.__sandbox_config = self.__extract_sandbox_config(meta)
...	...	@@ -2125,7 +2125,7 @@ class Element(Plugin):
2125	2125	# supports it.
2126	2126	#
2127	2127	def __use_remote_execution(self):
2128		- return self.__remote_execution_url and self.BST_VIRTUAL_DIRECTORY
	2128	+ return self.__remote_execution_specs and self.BST_VIRTUAL_DIRECTORY
2129	2129
2130	2130	# __sandbox():
2131	2131	#
...	...	@@ -2160,13 +2160,13 @@ class Element(Plugin):
2160	2160	stdout=stdout,
2161	2161	stderr=stderr,
2162	2162	config=config,
2163		- server_url=self.__remote_execution_url,
	2163	+ specs=self.__remote_execution_specs,
2164	2164	bare_directory=bare_directory,
2165	2165	allow_real_directory=False)
2166	2166	yield sandbox
2167	2167
2168	2168	elif directory is not None and os.path.exists(directory):
2169		- if allow_remote and self.__remote_execution_url:
	2169	+ if allow_remote and self.__remote_execution_specs:
2170	2170	self.warn("Artifact {} is configured to use remote execution but element plugin does not support it."
2171	2171	.format(self.name), detail="Element plugin '{kind}' does not support virtual directories."
2172	2172	.format(kind=self.get_kind()), warning_token="remote-failure")

buildstream/sandbox/_sandboxremote.py

---

...	...	@@ -19,6 +19,7 @@
19	19	# Jim MacArthur <jim macarthur codethink co uk>
20	20
21	21	import os
	22	+from collections import namedtuple
22	23	from urllib.parse import urlparse
23	24	from functools import partial
24	25
...	...	@@ -31,7 +32,15 @@ from .. import _signals
31	32	from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
32	33	from .._protos.google.rpc import code_pb2
33	34	from .._exceptions import SandboxError
	35	+from .. import _yaml
34	36	from .._protos.google.longrunning import operations_pb2, operations_pb2_grpc
	37	+from .._artifactcache.cascache import CASRemote, CASRemoteSpec
	38	+
	39	+from .._exceptions import SandboxError
	40	+
	41	+
	42	+class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service')):
	43	+ pass
35	44
36	45
37	46	# SandboxRemote()
...	...	@@ -44,17 +53,67 @@ class SandboxRemote(Sandbox):
44	53	def __init__(self, *args, **kwargs):
45	54	super().__init__(*args, **kwargs)
46	55
47		- url = urlparse(kwargs['server_url'])
48		- if not url.scheme or not url.hostname or not url.port:
49		- raise SandboxError("Configured remote URL '{}' does not match the expected layout. "
50		- .format(kwargs['server_url']) +
51		- "It should be of the form <protocol>://<domain name>:<port>.")
52		- elif url.scheme != 'http':
53		- raise SandboxError("Configured remote '{}' uses an unsupported protocol. "
54		- "Only plain HTTP is currenlty supported (no HTTPS).")
	56	+ config = kwargs['specs'] # This should be a RemoteExecutionSpec
	57	+ if config is None:
	58	+ return
55	59
56		- self.server_url = '{}:{}'.format(url.hostname, url.port)
57		- self.operation_name = None
	60	+ self.storage_url = config.storage_service['url']
	61	+ self.exec_url = config.exec_service['url']
	62	+
	63	+ self.storage_remote_spec = CASRemoteSpec(self.storage_url, push=True,
	64	+ server_cert=config.storage_service['server-cert'],
	65	+ client_key=config.storage_service['client-key'],
	66	+ client_cert=config.storage_service['client-cert'])
	67	+
	68	+ @staticmethod
	69	+ def specs_from_config_node(config_node, basedir):
	70	+
	71	+ def require_node(config, keyname):
	72	+ val = config.get(keyname)
	73	+ if val is None:
	74	+ provenance = _yaml.node_get_provenance(remote_config, key=keyname)
	75	+ raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
	76	+ "'{}' was not present in the remote "
	77	+ "execution configuration (remote-execution). "
	78	+ .format(keyname))
	79	+ return val
	80	+
	81	+ remote_config = config_node.get("remote-execution", None)
	82	+ if remote_config is None:
	83	+ return None
	84	+
	85	+ # Maintain some backwards compatibility with older configs, in which 'url' was the only valid key for
	86	+ # remote-execution.
	87	+
	88	+ tls_keys = ['client-key', 'client-cert', 'server-cert']
	89	+
	90	+ _yaml.node_validate(remote_config, ['execution-service', 'storage-service', 'url'])
	91	+ remote_exec_service_config = require_node(remote_config, 'execution-service')
	92	+ remote_exec_storage_config = require_node(remote_config, 'storage-service')
	93	+
	94	+ _yaml.node_validate(remote_exec_service_config, ['url'])
	95	+ _yaml.node_validate(remote_exec_storage_config, ['url'] + tls_keys)
	96	+
	97	+ if 'url' in remote_config:
	98	+ if 'execution-service' not in remote_config:
	99	+ remote_config['execution-service'] = {'url': remote_config['url']}
	100	+ else:
	101	+ provenance = _yaml.node_get_provenance(remote_config, key='url')
	102	+ raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
	103	+ "'url' and 'execution-service' keys were found in the remote "
	104	+ "execution configuration (remote-execution). "
	105	+ "You can only specify one of these.")
	106	+
	107	+ for key in tls_keys:
	108	+ if key not in remote_exec_storage_config:
	109	+ provenance = _yaml.node_get_provenance(remote_config, key='storage-service')
	110	+ raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
	111	+ "{}: The keys {} are necessary for the storage-service section of "
	112	+ "remote-execution configuration. Your config is missing '{}'."
	113	+ .format(str(provenance), tls_keys, key))
	114	+
	115	+ spec = RemoteExecutionSpec(remote_config['execution-service'], remote_config['storage-service'])
	116	+ return spec
58	117
59	118	def run_remote_command(self, command, input_root_digest, working_directory, environment):
60	119	# Sends an execution request to the remote execution server.
...	...	@@ -73,12 +132,13 @@ class SandboxRemote(Sandbox):
73	132	output_directories=[self._output_directory],
74	133	platform=None)
75	134	context = self._get_context()
76		- cascache = context.artifactcache
	135	+ cascache = context.get_cascache()
	136	+ casremote = CASRemote(self.storage_remote_spec)
	137	+
77	138	# Upload the Command message to the remote CAS server
78		- command_digest = cascache.push_message(self._get_project(), remote_command)
79		- if not command_digest or not cascache.verify_digest_pushed(self._get_project(), command_digest):
	139	+ command_digest = cascache.push_message(casremote, remote_command)
	140	+ if not command_digest or not cascache.verify_digest_on_remote(casremote, command_digest):
80	141	raise SandboxError("Failed pushing build command to remote CAS.")
81		-
82	142	# Create and send the action.
83	143	action = remote_execution_pb2.Action(command_digest=command_digest,
84	144	input_root_digest=input_root_digest,
...	...	@@ -86,15 +146,25 @@ class SandboxRemote(Sandbox):
86	146	do_not_cache=False)
87	147
88	148	# Upload the Action message to the remote CAS server
89		- action_digest = cascache.push_message(self._get_project(), action)
90		- if not action_digest or not cascache.verify_digest_pushed(self._get_project(), action_digest):
	149	+ action_digest = cascache.push_message(casremote, action)
	150	+ if not action_digest or not cascache.verify_digest_on_remote(casremote, action_digest):
91	151	raise SandboxError("Failed pushing build action to remote CAS.")
92	152
93	153	# Next, try to create a communication channel to the BuildGrid server.
94		- channel = grpc.insecure_channel(self.server_url)
	154	+ url = urlparse(self.exec_url)
	155	+ if not url.port:
	156	+ raise SandboxError("You must supply a protocol and port number in the execution-service url, "
	157	+ "for example: http://buildservice:50051.")
	158	+ if url.scheme == 'http':
	159	+ channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
	160	+ else:
	161	+ raise SandboxError("Remote execution currently only supports the 'http' protocol "
	162	+ "and '{}' was supplied.".format(url.scheme))
	163	+
95	164	stub = remote_execution_pb2_grpc.ExecutionStub(channel)
96	165	request = remote_execution_pb2.ExecuteRequest(action_digest=action_digest,
97	166	skip_cache_lookup=False)
	167	+ self.operation_name = None
98	168
99	169	def __run_remote_command(stub, execute_request=None, running_operation=None):
100	170	try:
...	...	@@ -117,7 +187,7 @@ class SandboxRemote(Sandbox):
117	187	status_code = e.code()
118	188	if status_code == grpc.StatusCode.UNAVAILABLE:
119	189	raise SandboxError("Failed contacting remote execution server at {}."
120		- .format(self.server_url))
	190	+ .format(self.exec_url))
121	191
122	192	elif status_code in (grpc.StatusCode.INVALID_ARGUMENT,
123	193	grpc.StatusCode.FAILED_PRECONDITION,
...	...	@@ -188,9 +258,11 @@ class SandboxRemote(Sandbox):
188	258	raise SandboxError("Output directory structure had no digest attached.")
189	259
190	260	context = self._get_context()
191		- cascache = context.artifactcache
	261	+ cascache = context.get_cascache()
	262	+ casremote = CASRemote(self.storage_remote_spec)
	263	+
192	264	# Now do a pull to ensure we have the necessary parts.
193		- dir_digest = cascache.pull_tree(self._get_project(), tree_digest)
	265	+ dir_digest = cascache.pull_tree(casremote, tree_digest)
194	266	if dir_digest is None or not dir_digest.hash or not dir_digest.size_bytes:
195	267	raise SandboxError("Output directory structure pulling from remote failed.")
196	268
...	...	@@ -216,18 +288,23 @@ class SandboxRemote(Sandbox):
216	288	# Upload sources
217	289	upload_vdir = self.get_virtual_directory()
218	290
	291	+ cascache = self._get_context().get_cascache()
219	292	if isinstance(upload_vdir, FileBasedDirectory):
220	293	# Make a new temporary directory to put source in
221		- upload_vdir = CasBasedDirectory(self._get_context().artifactcache.cas, ref=None)
	294	+ upload_vdir = CasBasedDirectory(cascache, ref=None)
222	295	upload_vdir.import_files(self.get_virtual_directory()._get_underlying_directory())
223	296
224	297	upload_vdir.recalculate_hash()
225	298
226		- context = self._get_context()
227		- cascache = context.artifactcache
	299	+ casremote = CASRemote(self.storage_remote_spec)
228	300	# Now, push that key (without necessarily needing a ref) to the remote.
229		- cascache.push_directory(self._get_project(), upload_vdir)
230		- if not cascache.verify_digest_pushed(self._get_project(), upload_vdir.ref):
	301	+
	302	+ try:
	303	+ cascache.push_directory(casremote, upload_vdir)
	304	+ except grpc._channel._Rendezvous as e:
	305	+ raise SandboxError("Failed to push source directory to remote: {}".format(e)) from e
	306	+
	307	+ if not cascache.verify_digest_on_remote(casremote, upload_vdir.ref):
231	308	raise SandboxError("Failed to verify that source has been pushed to the remote artifact cache.")
232	309
233	310	# Fallback to the sandbox default settings for

doc/source/format_project.rst

---

...	...	@@ -231,10 +231,24 @@ using the `remote-execution` option:
231	231	remote-execution:
232	232
233	233	# A url defining a remote execution server
234		- url: http://buildserver.example.com:50051
	234	+ execution-service:
	235	+ url: http://buildserver.example.com:50051
	236	+ storage-service:
	237	+ - url: https://foo.com/artifacts:11002
	238	+ server-cert: server.crt
	239	+ client-cert: client.crt
	240	+ client-key: client.key
	241	+
	242	+The execution-server part of remote execution does not support encrypted
	243	+connections yet, so the protocol must always be http.
	244	+
	245	+storage-server specifies a remote CAS store and the parameters are the
	246	+same as those used to specify an :ref:`artifact server <artifacts>`.
235	247
236		-The url should contain a hostname and port separated by ':'. Only plain HTTP is
237		-currently suported (no HTTPS).
	248	+The storage server may be the same server used for artifact
	249	+caching. Remote execution cannot work without push access to the
	250	+storage server, so you must specify a client certificate and key, and
	251	+a server certificate.
238	252
239	253	The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
240	254

tests/sandboxes/remote-exec-config.py

---

	1	+import pytest
	2	+
	3	+import itertools
	4	+import os
	5	+
	6	+from buildstream import _yaml
	7	+from buildstream._exceptions import ErrorDomain, LoadErrorReason
	8	+
	9	+from tests.testutils.runcli import cli
	10	+
	11	+DATA_DIR = os.path.dirname(os.path.realpath(__file__))
	12	+
	13	+# Tests that we get a useful error message when supplying invalid
	14	+# remote execution configurations.
	15	+
	16	+
	17	+# Assert that if both 'url' (the old style) and 'execution-service' (the new style)
	18	+# are used at once, a LoadError results.
	19	+@pytest.mark.datafiles(DATA_DIR)
	20	+def test_old_and_new_configs(cli, datafiles):
	21	+ project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
	22	+
	23	+ project_conf = {
	24	+ 'name': 'test',
	25	+
	26	+ 'remote-execution': {
	27	+ 'url': 'https://cache.example.com:12345',
	28	+ 'execution-service': {
	29	+ 'url': 'http://localhost:8088'
	30	+ },
	31	+ 'storage-service': {
	32	+ 'url': 'http://charactron:11001',
	33	+ }
	34	+ }
	35	+ }
	36	+ project_conf_file = os.path.join(project, 'project.conf')
	37	+ _yaml.dump(project_conf, project_conf_file)
	38	+
	39	+ # Use `pull` here to ensure we try to initialize the remotes, triggering the error
	40	+ #
	41	+ # This does not happen for a simple `bst show`.
	42	+ result = cli.run(project=project, args=['pull', 'element.bst'])
	43	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")
	44	+
	45	+
	46	+# Assert that if either the client key or client cert is specified
	47	+# without specifying its counterpart, we get a comprehensive LoadError
	48	+# instead of an unhandled exception.
	49	+@pytest.mark.datafiles(DATA_DIR)
	50	+@pytest.mark.parametrize('config_key, config_value', [
	51	+ ('client-cert', 'client.crt'),
	52	+ ('client-key', 'client.key')
	53	+])
	54	+def test_missing_certs(cli, datafiles, config_key, config_value):
	55	+ project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
	56	+
	57	+ project_conf = {
	58	+ 'name': 'test',
	59	+
	60	+ 'remote-execution': {
	61	+ 'execution-service': {
	62	+ 'url': 'http://localhost:8088'
	63	+ },
	64	+ 'storage-service': {
	65	+ 'url': 'http://charactron:11001',
	66	+ config_key: config_value,
	67	+ }
	68	+ }
	69	+ }
	70	+ project_conf_file = os.path.join(project, 'project.conf')
	71	+ _yaml.dump(project_conf, project_conf_file)
	72	+
	73	+ # Use `pull` here to ensure we try to initialize the remotes, triggering the error
	74	+ #
	75	+ # This does not happen for a simple `bst show`.
	76	+ result = cli.run(project=project, args=['show', 'element.bst'])
	77	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "Your config is missing")
	78	+
	79	+
	80	+# Assert that if incomplete information is supplied we get a sensible error message.
	81	+@pytest.mark.datafiles(DATA_DIR)
	82	+def test_empty_config(cli, datafiles):
	83	+ project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
	84	+
	85	+ project_conf = {
	86	+ 'name': 'test',
	87	+
	88	+ 'remote-execution': {
	89	+ }
	90	+ }
	91	+ project_conf_file = os.path.join(project, 'project.conf')
	92	+ _yaml.dump(project_conf, project_conf_file)
	93	+
	94	+ # Use `pull` here to ensure we try to initialize the remotes, triggering the error
	95	+ #
	96	+ # This does not happen for a simple `bst show`.
	97	+ result = cli.run(project=project, args=['pull', 'element.bst'])
	98	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.