[Notes] [Git][BuildGrid/buildgrid][santi/fetch-dir-batch-read] 2 commits: test_client: add subdirectories to data/

[Santiago Gil <gitlab mg gitlab com>]

Title: GitLab

Santiago Gil pushed to branch santi/fetch-dir-batch-read at BuildGrid / buildgrid

Commits:

• 0f5db63a
  by Santiago Gil at 2019-01-22T09:29:37Z

  test_client: add subdirectories to data/
  

• 2248a358
  by Santiago Gil at 2019-01-22T09:29:37Z

  Downloader._write_directory(): request dirs in batches
  
  Attempt to fetch directories in a given level using a single batch
  request. If not possible to do so, fall back to using Read().
  

4 changed files:

• buildgrid/client/cas.py
• + tests/cas/data/includes/contrib/hello-contrib.h
• + tests/cas/data/includes/hello1.h
• tests/cas/test_client.py

Changes:

buildgrid/client/cas.py

---

...	...	@@ -24,7 +24,7 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p
24	24	from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
25	25	from buildgrid._protos.google.rpc import code_pb2
26	26	from buildgrid.settings import HASH, MAX_REQUEST_SIZE, MAX_REQUEST_COUNT
27		-from buildgrid.utils import merkle_tree_maker
	27	+from buildgrid.utils import create_digest, merkle_tree_maker
28	28
29	29
30	30	# Maximum size for a queueable file:
...	...	@@ -409,26 +409,54 @@ class Downloader:
409	409	self._write_directory(directory, directory_path,
410	410	root_barrier=directory_path)
411	411
412		- def _write_directory(self, root_directory, root_path, directories=None, root_barrier=None):
	412	+ def _write_directory(self, root_directory, root_path, directories=None,
	413	+ root_barrier=None):
413	414	"""Generates a local directory structure"""
	415	+
	416	+ # i) Files:
414	417	for file_node in root_directory.files:
415	418	file_path = os.path.join(root_path, file_node.name)
416	419
417		- self._queue_file(file_node.digest, file_path, is_executable=file_node.is_executable)
	420	+ self._queue_file(file_node.digest, file_path,
	421	+ is_executable=file_node.is_executable)
418	422
	423	+ # ii) Directories:
	424	+ pending_directory_digests = []
	425	+ pending_directory_paths = {}
419	426	for directory_node in root_directory.directories:
	427	+ directory_hash = directory_node.digest.hash
	428	+
420	429	directory_path = os.path.join(root_path, directory_node.name)
	430	+ os.makedirs(directory_path, exist_ok=True)
	431	+
421	432	if directories and directory_node.digest.hash in directories:
422		- directory = directories[directory_node.digest.hash]
	433	+ # We already have the directory; just write it:
	434	+ directory = directories[directory_hash]
	435	+
	436	+ self._write_directory(directory, directory_path,
	437	+ directories=directories,
	438	+ root_barrier=root_barrier)
423	439	else:
	440	+ # Gather all the directories that we need to fetch to
	441	+ # try getting them in a single batch request:
	442	+ pending_directory_digests.append(directory_node.digest)
	443	+ pending_directory_paths[directory_hash] = directory_path
	444	+
	445	+ if pending_directory_paths:
	446	+ fetched_blobs = self._fetch_blob_batch(pending_directory_digests)
	447	+
	448	+ for directory_blob in fetched_blobs:
424	449	directory = remote_execution_pb2.Directory()
425		- directory.ParseFromString(self._fetch_blob(directory_node.digest))
	450	+ directory.ParseFromString(directory_blob)
426	451
427		- os.makedirs(directory_path, exist_ok=True)
	452	+ directory_hash = create_digest(directory_blob).hash
	453	+ directory_path = pending_directory_paths[directory_hash]
428	454
429		- self._write_directory(directory, directory_path,
430		- directories=directories, root_barrier=root_barrier)
	455	+ self._write_directory(directory, directory_path,
	456	+ directories=directories,
	457	+ root_barrier=root_barrier)
431	458
	459	+ # iii) Symlinks:
432	460	for symlink_node in root_directory.symlinks:
433	461	symlink_path = os.path.join(root_path, symlink_node.name)
434	462	if not os.path.isabs(symlink_node.target):

tests/cas/data/includes/contrib/hello-contrib.h

---

	1	+/* Hello from the contrib/ directory */
	2	+#define HELLO_CONTRIB "Hello!"

tests/cas/data/includes/hello1.h

---

	1	+/* Hello1.h */
	2	+#define HELLO1 "Hello1!"

tests/cas/test_client.py

---

...	...	@@ -44,12 +44,17 @@ FILES = [
44	44	(os.path.join(DATA_DIR, 'void'),),
45	45	(os.path.join(DATA_DIR, 'hello.cc'),),
46	46	(os.path.join(DATA_DIR, 'hello', 'hello.c'),
47		- os.path.join(DATA_DIR, 'hello', 'hello.h'))]
	47	+ os.path.join(DATA_DIR, 'hello', 'hello.h'),
	48	+ os.path.join(DATA_DIR, 'hello', 'hello.sh'),),
	49	+ (os.path.join(DATA_DIR, 'includes', 'hello1.h'),),
	50	+ (os.path.join(DATA_DIR, 'includes', 'contrib', 'hello-contrib.h'),)]
48	51	FOLDERS = [
49		- (os.path.join(DATA_DIR, 'hello'),)]
	52	+ (os.path.join(DATA_DIR, 'hello'),),
	53	+ (os.path.join(DATA_DIR, 'includes'),),
	54	+ (os.path.join(DATA_DIR, 'includes', 'contrib'),)]
50	55	DIRECTORIES = [
51	56	(os.path.join(DATA_DIR, 'hello'),),
52		- (os.path.join(DATA_DIR, 'hello'), DATA_DIR)]
	57	+ (os.path.join(DATA_DIR, 'includes'),)]
53	58
54	59
55	60	@pytest.mark.parametrize('blobs', BLOBS)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.