[Notes] [Git][BuildStream/buildstream][valentindavid/remote_execution_configuration] 5 commits: sandbox/sandbox.py: Use assertions for programming errors instead of BstErrors.

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/remote_execution_configuration at BuildStream / buildstream

Commits:

• 81f7a964
  by Tristan Van Berkom at 2019-01-08T16:55:58Z

  sandbox/sandbox.py: Use assertions for programming errors instead of BstErrors.
  
  When code is faulty, in a plugin or in the core, we should
  get a stack trace and a BUG message.
  

• edcc43ed
  by Tristan Van Berkom at 2019-01-08T19:19:10Z

  Merge branch 'tristan/sandbox-programming-error' into 'master'
  
  sandbox/sandbox.py: Use assertions for programming errors instead of BstErrors.
  
  See merge request BuildStream/buildstream!1046

• 17c5ca2d
  by Valentin David at 2019-01-09T10:45:23Z

  Use relative path to project directory for remote execution certificates/keys
  

• 605f8d11
  by Valentin David at 2019-01-09T10:45:23Z

  Add support for user remote execution configuration
  
  Fixes #631.
  

• 65ea03ab
  by Valentin David at 2019-01-09T10:45:23Z

  Add support for https channel to remote execution and actions servers
  
  Fixes #780.
  

6 changed files:

• buildstream/_context.py
• buildstream/_project.py
• buildstream/sandbox/_sandboxremote.py
• buildstream/sandbox/sandbox.py
• doc/source/format_project.rst
• doc/source/using_config.rst

Changes:

buildstream/_context.py

---

...	...	@@ -34,6 +34,7 @@ from ._artifactcache import ArtifactCache
34	34	from ._artifactcache.cascache import CASCache
35	35	from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
36	36	from .plugin import _plugin_lookup
	37	+from .sandbox import SandboxRemote
37	38
38	39
39	40	# Context()
...	...	@@ -72,6 +73,9 @@ class Context():
72	73	# The locations from which to push and pull prebuilt artifacts
73	74	self.artifact_cache_specs = None
74	75
	76	+ # The global remote execution configuration
	77	+ self.remote_execution_specs = None
	78	+
75	79	# The directory to store build logs
76	80	self.logdir = None
77	81
...	...	@@ -187,7 +191,7 @@ class Context():
187	191	_yaml.node_validate(defaults, [
188	192	'sourcedir', 'builddir', 'artifactdir', 'logdir',
189	193	'scheduler', 'artifacts', 'logging', 'projects',
190		- 'cache', 'prompt', 'workspacedir',
	194	+ 'cache', 'prompt', 'workspacedir', 'remote-execution'
191	195	])
192	196
193	197	for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
...	...	@@ -212,6 +216,8 @@ class Context():
212	216	# Load artifact share configuration
213	217	self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
214	218
	219	+ self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
	220	+
215	221	# Load pull build trees configuration
216	222	self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
217	223
...	...	@@ -271,7 +277,8 @@ class Context():
271	277	# Shallow validation of overrides, parts of buildstream which rely
272	278	# on the overrides are expected to validate elsewhere.
273	279	for _, overrides in _yaml.node_items(self._project_overrides):
274		- _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror'])
	280	+ _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror',
	281	+ 'remote-execution'])
275	282
276	283	profile_end(Topics.LOAD_CONTEXT, 'load')
277	284

buildstream/_project.py

---

...	...	@@ -507,7 +507,16 @@ class Project():
507	507	self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
508	508
509	509	# Load remote-execution configuration for this project
510		- self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
	510	+ project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
	511	+ override_specs = SandboxRemote.specs_from_config_node(
	512	+ self._context.get_overrides(self.name), self.directory)
	513	+
	514	+ if override_specs is not None:
	515	+ self.remote_execution_specs = override_specs
	516	+ elif project_specs is not None:
	517	+ self.remote_execution_specs = project_specs
	518	+ else:
	519	+ self.remote_execution_specs = self._context.remote_execution_specs
511	520
512	521	# Load sandbox environment variables
513	522	self.base_environment = _yaml.node_get(config, Mapping, 'environment')

buildstream/sandbox/_sandboxremote.py

---

...	...	@@ -62,10 +62,32 @@ class SandboxRemote(Sandbox):
62	62	self.storage_url = config.storage_service['url']
63	63	self.exec_url = config.exec_service['url']
64	64
	65	+ exec_certs = {}
	66	+ for key in ['client-cert', 'client-key', 'server-cert']:
	67	+ if key in config.exec_service:
	68	+ with open(config.exec_service[key], 'rb') as f:
	69	+ exec_certs[key] = f.read()
	70	+
	71	+ self.exec_credentials = grpc.ssl_channel_credentials(
	72	+ root_certificates=exec_certs.get('server-cert'),
	73	+ private_key=exec_certs.get('client-key'),
	74	+ certificate_chain=exec_certs.get('client-cert'))
	75	+
	76	+ action_certs = {}
	77	+ for key in ['client-cert', 'client-key', 'server-cert']:
	78	+ if key in config.action_service:
	79	+ with open(config.action_service[key], 'rb') as f:
	80	+ action_certs[key] = f.read()
	81	+
65	82	if config.action_service:
66	83	self.action_url = config.action_service['url']
	84	+ self.action_credentials = grpc.ssl_channel_credentials(
	85	+ root_certificates=action_certs.get('server-cert'),
	86	+ private_key=action_certs.get('client-key'),
	87	+ certificate_chain=action_certs.get('client-cert'))
67	88	else:
68	89	self.action_url = None
	90	+ self.action_credentials = None
69	91
70	92	self.server_instance = config.exec_service.get('instance', None)
71	93	self.storage_instance = config.storage_service.get('instance', None)
...	...	@@ -81,7 +103,7 @@ class SandboxRemote(Sandbox):
81	103	self._get_context().message(Message(None, MessageType.INFO, msg))
82	104
83	105	@staticmethod
84		- def specs_from_config_node(config_node, basedir):
	106	+ def specs_from_config_node(config_node, basedir=None):
85	107
86	108	def require_node(config, keyname):
87	109	val = config.get(keyname)
...	...	@@ -109,10 +131,10 @@ class SandboxRemote(Sandbox):
109	131	remote_exec_storage_config = require_node(remote_config, 'storage-service')
110	132	remote_exec_action_config = remote_config.get('action-cache-service', {})
111	133
112		- _yaml.node_validate(remote_exec_service_config, ['url', 'instance'])
	134	+ _yaml.node_validate(remote_exec_service_config, ['url', 'instance'] + tls_keys)
113	135	_yaml.node_validate(remote_exec_storage_config, ['url', 'instance'] + tls_keys)
114	136	if remote_exec_action_config:
115		- _yaml.node_validate(remote_exec_action_config, ['url'])
	137	+ _yaml.node_validate(remote_exec_action_config, ['url'] + tls_keys)
116	138	else:
117	139	remote_config['action-service'] = None
118	140
...	...	@@ -135,6 +157,19 @@ class SandboxRemote(Sandbox):
135	157	"remote-execution configuration. Your config is missing '{}'."
136	158	.format(str(provenance), tls_keys, key))
137	159
	160	+ def resolve_path(path):
	161	+ if basedir and path:
	162	+ return os.path.join(basedir, path)
	163	+ else:
	164	+ return path
	165	+
	166	+ for key in tls_keys:
	167	+ for d in (remote_config['execution-service'],
	168	+ remote_config['storage-service'],
	169	+ remote_exec_action_config):
	170	+ if key in d:
	171	+ d[key] = resolve_path(d[key])
	172	+
138	173	spec = RemoteExecutionSpec(remote_config['execution-service'],
139	174	remote_config['storage-service'],
140	175	remote_exec_action_config)
...	...	@@ -295,6 +330,8 @@ class SandboxRemote(Sandbox):
295	330	"for example: http://buildservice:50051.")
296	331	if url.scheme == 'http':
297	332	channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
	333	+ elif url.scheme == 'https':
	334	+ channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.exec_credentials)
298	335	else:
299	336	raise SandboxError("Remote execution currently only supports the 'http' protocol "
300	337	"and '{}' was supplied.".format(url.scheme))
...	...	@@ -352,11 +389,11 @@ class SandboxRemote(Sandbox):
352	389	if not url.port:
353	390	raise SandboxError("You must supply a protocol and port number in the action-cache-service url, "
354	391	"for example: http://buildservice:50051.")
355		- if not url.scheme == "http":
356		- raise SandboxError("Currently only support http for the action cache"
357		- "and {} was supplied".format(url.scheme))
	392	+ if url.scheme == 'http':
	393	+ channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
	394	+ elif url.scheme == 'https':
	395	+ channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.action_credentials)
358	396
359		- channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
360	397	request = remote_execution_pb2.GetActionResultRequest(action_digest=action_digest)
361	398	stub = remote_execution_pb2_grpc.ActionCacheStub(channel)
362	399	try:

buildstream/sandbox/sandbox.py

---

...	...	@@ -288,8 +288,8 @@ class Sandbox():
288	288	command = [command]
289	289
290	290	if self.__batch:
291		- if flags != self.__batch.flags:
292		- raise SandboxError("Inconsistent sandbox flags in single command batch")
	291	+ assert flags == self.__batch.flags, \
	292	+ "Inconsistent sandbox flags in single command batch"
293	293
294	294	batch_command = _SandboxBatchCommand(command, cwd=cwd, env=env, label=label)
295	295
...	...	@@ -326,8 +326,8 @@ class Sandbox():
326	326
327	327	if self.__batch:
328	328	# Nested batch
329		- if flags != self.__batch.flags:
330		- raise SandboxError("Inconsistent sandbox flags in single command batch")
	329	+ assert flags == self.__batch.flags, \
	330	+ "Inconsistent sandbox flags in single command batch"
331	331
332	332	parent_group = self.__batch.current_group
333	333	parent_group.append(group)

doc/source/format_project.rst

---

...	...	@@ -218,6 +218,7 @@ The use of ports are required to distinguish between pull only access and
218	218	push/pull access. For information regarding the server/client certificates
219	219	and keys, please see: :ref:`Key pair for the server <server_authentication>`.
220	220
	221	+.. _project_remote_execution:
221	222
222	223	Remote execution
223	224	~~~~~~~~~~~~~~~~
...	...	@@ -243,9 +244,6 @@ using the `remote-execution` option:
243	244	action-cache-service:
244	245	url: http://bar.action.com:50052
245	246
246		-The execution-service part of remote execution does not support encrypted
247		-connections yet, so the protocol must always be http.
248		-
249	247	storage-service specifies a remote CAS store and the parameters are the
250	248	same as those used to specify an :ref:`artifact server <artifacts>`.
251	249
...	...	@@ -268,6 +266,9 @@ instance names.
268	266
269	267	The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
270	268
	269	+Remote execution configuration can be also provided in the `user
	270	+configuration <user_config_remote_execution>`.
	271	+
271	272	.. _project_essentials_mirrors:
272	273
273	274	Mirrors

doc/source/using_config.rst

---

...	...	@@ -100,6 +100,54 @@ pull only access and push/pull access. For information regarding this and the
100	100	server/client certificates and keys, please see:
101	101	:ref:`Key pair for the server <server_authentication>`.
102	102
	103	+.. _user_config_remote_execution:
	104	+
	105	+Remote execution
	106	+~~~~~~~~~~~~~~~~
	107	+
	108	+The same configuration for :ref:`remote execution <project_remote_execution>`
	109	+in ``project.conf`` can be provided in the user configuation.
	110	+
	111	+There is only one remote execution configuration used per project.
	112	+
	113	+The project overrides will be taken in priority. The global
	114	+configuration will be used as fallback.
	115	+
	116	+1. Global remote execution fallback:
	117	+
	118	+.. code:: yaml
	119	+
	120	+ remote-execution:
	121	+ execution-service:
	122	+ url: http://execution.fallback.example.com:50051
	123	+ instance-name: main
	124	+ storage-service:
	125	+ url: https://storage.fallback.example.com:11002/
	126	+ server-cert: /keys/server.crt
	127	+ client-cert: /keys/client.crt
	128	+ client-key: /keys/client.key
	129	+ instance-name: main
	130	+ action-cache-service:
	131	+ url: http://action.flalback.example.com:50052
	132	+
	133	+2. Project override:
	134	+
	135	+.. code:: yaml
	136	+
	137	+ projects:
	138	+ some_project:
	139	+ remote-execution:
	140	+ execution-service:
	141	+ url: http://execution.some_project.example.com:50051
	142	+ instance-name: main
	143	+ storage-service:
	144	+ url: https://storage.some_project.example.com:11002/
	145	+ server-cert: /some_project_keys/server.crt
	146	+ client-cert: /some_project_keys/client.crt
	147	+ client-key: /some_project_keys/client.key
	148	+ instance-name: main
	149	+ action-cache-service:
	150	+ url: http://action.some_project.example.com:50052
103	151
104	152
105	153	Strict build plan

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.