[Notes] [Git][BuildGrid/buildgrid][finn/80-remote-parser] 3 commits: Added new config files and a missing `__init__.py`

[finn <gitlab mg gitlab com>]

Title: GitLab

finn pushed to branch finn/80-remote-parser at BuildGrid / buildgrid

Commits:

• 592e1af4
  by finnball at 2018-09-12T13:29:14Z

  Added new config files and a missing `__init__.py`
  
  Also moved location of a variable in `default.yml`
  

• 9e6ca3c9
  by finnball at 2018-09-12T13:33:17Z

  When raising error, correct variable is logged.
  

• 802c7649
  by finnball at 2018-09-12T13:33:17Z

  If blob not found, correctly handles exception.
  

6 changed files:

• + buildgrid/_app/settings/__init__.py
• + buildgrid/_app/settings/cas.yml
• buildgrid/_app/settings/default.yml
• + buildgrid/_app/settings/remote-storage.yml
• buildgrid/server/cas/service.py
• buildgrid/server/cas/storage/remote.py

Changes:

buildgrid/_app/settings/__init__.py

---

buildgrid/_app/settings/cas.yml

---

	1	+server:
	2	+ port: 50052
	3	+ insecure-mode: true
	4	+ tls-server-key: null
	5	+ tls-server-cert: null
	6	+ tls-client-certs: null
	7	+
	8	+description: |
	9	+ Just a CAS.
	10	+
	11	+instances:
	12	+ - name: main
	13	+ description: |
	14	+ The main server
	15	+
	16	+ storages:
	17	+ - !disk-storage &main-storage
	18	+ path: ~/cas/
	19	+
	20	+ services:
	21	+ - !cas
	22	+ storage: *main-storage
	23	+
	24	+ - !bytestream
	25	+ storage: *main-storage

buildgrid/_app/settings/default.yml

---

1	1	server:
2	2	port: 50051
	3	+ insecure-mode: true
3	4	tls-server-key: null
4	5	tls-server-cert: null
5	6	tls-client-certs: null
6		- insecure-mode: true
7	7
8	8	description: |
9	9	A single default instance

buildgrid/_app/settings/remote-storage.yml

---

	1	+server:
	2	+ port: 50051
	3	+ insecure-mode: true
	4	+ tls-server-key: null
	5	+ tls-server-cert: null
	6	+ tls-client-certs: null
	7	+
	8	+
	9	+description: |
	10	+ A single default instance with remote storage.
	11	+
	12	+instances:
	13	+ - name: main
	14	+ description: |
	15	+ The main server
	16	+
	17	+ storages:
	18	+ - !remote-storage &main-storage
	19	+ url: "http://localhost:50052"
	20	+ credentials:
	21	+ tls-client-key: null
	22	+ tls-client-cert: null
	23	+ tls-server-cert: null
	24	+
	25	+ services:
	26	+ - !action-cache &main-action
	27	+ storage: *main-storage
	28	+ max_cached_refs: 256
	29	+ allow_updates: true
	30	+
	31	+ - !execution
	32	+ storage: *main-storage
	33	+ action_cache: *main-action
	34	+
	35	+ - !cas
	36	+ storage: *main-storage
	37	+
	38	+ - !bytestream
	39	+ storage: *main-storage

buildgrid/server/cas/service.py

---

...	...	@@ -89,15 +89,15 @@ class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
89	89	# TODO: Decide on default instance name
90	90	if path[0] == "blobs":
91	91	if len(path) < 3 or not path[2].isdigit():
92		- raise InvalidArgumentError("Invalid resource name: {}".format(context.resource_name))
	92	+ raise InvalidArgumentError("Invalid resource name: {}".format(request.resource_name))
93	93	instance_name = ""
94	94
95	95	elif path[1] == "blobs":
96	96	if len(path) < 4 or not path[3].isdigit():
97		- raise InvalidArgumentError("Invalid resource name: {}".format(context.resource_name))
	97	+ raise InvalidArgumentError("Invalid resource name: {}".format(request.resource_name))
98	98
99	99	else:
100		- raise InvalidArgumentError("Invalid resource name: {}".format(context.resource_name))
	100	+ raise InvalidArgumentError("Invalid resource name: {}".format(request.resource_name))
101	101
102	102	instance = self._get_instance(instance_name)
103	103	yield from instance.read(path,
...	...	@@ -134,15 +134,15 @@ class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
134	134	# TODO: Sort out no instance name
135	135	if path[0] == "uploads":
136	136	if len(path) < 5 or path[2] != "blobs" or not path[4].isdigit():
137		- raise InvalidArgumentError("Invalid resource name: {}".format(context.resource_name))
	137	+ raise InvalidArgumentError("Invalid resource name: {}".format(first_request.resource_name))
138	138	instance_name = ""
139	139
140	140	elif path[1] == "uploads":
141	141	if len(path) < 6 or path[3] != "blobs" or not path[5].isdigit():
142		- raise InvalidArgumentError("Invalid resource name: {}".format(context.resource_name))
	142	+ raise InvalidArgumentError("Invalid resource name: {}".format(first_request.resource_name))
143	143
144	144	else:
145		- raise InvalidArgumentError("Invalid resource name: {}".format(context.resource_name))
	145	+ raise InvalidArgumentError("Invalid resource name: {}".format(first_request.resource_name))
146	146
147	147	instance = self._get_instance(instance_name)
148	148	return instance.write(requests)

buildgrid/server/cas/storage/remote.py

---

...	...	@@ -23,6 +23,8 @@ Forwwards storage requests to a remote storage.
23	23	import io
24	24	import logging
25	25
	26	+import grpc
	27	+
26	28	from buildgrid.utils import gen_fetch_blob, gen_write_request_blob
27	29	from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
28	30	from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
...	...	@@ -44,18 +46,29 @@ class RemoteStorage(StorageABC):
44	46	return False
45	47
46	48	def get_blob(self, digest):
47		- fetched_data = io.BytesIO()
48		- length = 0
49		- for data in gen_fetch_blob(self._stub_bs, digest, self._instance_name):
50		- length += fetched_data.write(data)
51		-
52		- if length:
53		- assert digest.size_bytes == length
54		- fetched_data.seek(0)
55		- return fetched_data
56		-
57		- else:
58		- return None
	49	+ try:
	50	+ fetched_data = io.BytesIO()
	51	+ length = 0
	52	+
	53	+ for data in gen_fetch_blob(self._stub_bs, digest, self._instance_name):
	54	+ length += fetched_data.write(data)
	55	+
	56	+ if length:
	57	+ assert digest.size_bytes == length
	58	+ fetched_data.seek(0)
	59	+ return fetched_data
	60	+
	61	+ else:
	62	+ return None
	63	+
	64	+ except grpc.RpcError as e:
	65	+ if e.code() == grpc.StatusCode.NOT_FOUND:
	66	+ pass
	67	+ else:
	68	+ self.logger.error(e.details())
	69	+ raise
	70	+
	71	+ return None
59	72
60	73	def begin_write(self, digest):
61	74	return io.BytesIO(digest.SerializeToString())

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.