Arber Xhindoli pushed to branch arber/91-get-tree at BuildGrid / buildgrid
Commits:
-
3593108e
by Arber Xhindoli at 2018-11-15T22:40:21Z
3 changed files:
Changes:
| ... | ... | @@ -390,7 +390,7 @@ class Downloader: |
| 390 | 390 |
assert digest.hash in directories
|
| 391 | 391 |
|
| 392 | 392 |
directory = directories[digest.hash]
|
| 393 |
- self._write_directory(digest.hash, directory_path,
|
|
| 393 |
+ self._write_directory(directory, directory_path,
|
|
| 394 | 394 |
directories=directories, root_barrier=directory_path)
|
| 395 | 395 |
|
| 396 | 396 |
directory_fetched = True
|
| ... | ... | @@ -58,18 +58,26 @@ class ContentAddressableStorageInstance: |
| 58 | 58 |
|
| 59 | 59 |
return response
|
| 60 | 60 |
|
| 61 |
- def get_tree(self, request, directory_list, digest_list):
|
|
| 62 |
- """
|
|
| 63 |
- This function will start reading directories at request.root_digest.
|
|
| 64 |
- It will push the directories, and their corresponding digests into the
|
|
| 65 |
- directory_list and digest_list.
|
|
| 66 |
- |
|
| 67 |
- It will continue to do a level-order traversal until either: directory_list reaches the end,
|
|
| 68 |
- or we have made request.page_size reads. If the latter case, it will return len(directory_list) - 1 so
|
|
| 69 |
- subsequent calls can pick up where it left off.
|
|
| 70 |
- Otherwise, returns None, meaning we have read the directory tree.
|
|
| 71 |
- """
|
|
| 72 |
- return None
|
|
| 61 |
+ def get_tree(self, request):
|
|
| 62 |
+ storage = self._storage
|
|
| 63 |
+ |
|
| 64 |
+ # Create getTreeResponse message
|
|
| 65 |
+ # TODO: handle page_token, page_size
|
|
| 66 |
+ response = re_pb2.GetTreeResponse()
|
|
| 67 |
+ |
|
| 68 |
+ # Set to MAX_REQUEST_COUNT, will use in the future to limit size of GRPC response
|
|
| 69 |
+ if not request.page_size:
|
|
| 70 |
+ request.page_size = 500
|
|
| 71 |
+ |
|
| 72 |
+ def _get_tree(node_digest):
|
|
| 73 |
+ directory_from_digest = storage.get_message(node_digest, re_pb2.Directory)
|
|
| 74 |
+ directories = [directory_from_digest]
|
|
| 75 |
+ for directory in directory_from_digest.directories:
|
|
| 76 |
+ directories.extend(_get_tree(directory.digest))
|
|
| 77 |
+ return directories
|
|
| 78 |
+ |
|
| 79 |
+ response.directories.extend(_get_tree(request.root_digest))
|
|
| 80 |
+ yield response
|
|
| 73 | 81 |
|
| 74 | 82 |
|
| 75 | 83 |
class ByteStreamInstance:
|
| ... | ... | @@ -87,56 +87,14 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa |
| 87 | 87 |
def GetTree(self, request, context):
|
| 88 | 88 |
self.__logger.debug("GetTree request from [%s]", context.peer())
|
| 89 | 89 |
|
| 90 |
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
|
| 91 |
- context.set_details('Method not implemented!')
|
|
| 92 |
- |
|
| 93 |
- # Stores the directories as long as a page token is returned.
|
|
| 94 |
- directories = []
|
|
| 95 |
- # Stores the digests of those directories
|
|
| 96 |
- digests = []
|
|
| 97 |
- |
|
| 98 |
- # if page_size is not set
|
|
| 99 |
- if request.page_size == 0:
|
|
| 100 |
- request.page_size = 500
|
|
| 101 |
- |
|
| 102 |
- # Set to 0, will be used to index into directory list,
|
|
| 103 |
- # and updated in instance.get_tree. This is the only way this makes sense to me.
|
|
| 104 |
- request.page_token = 0
|
|
| 105 |
- |
|
| 106 |
- # start at index 1, to not return root
|
|
| 107 |
- start_index = 1
|
|
| 108 |
- |
|
| 109 | 90 |
try:
|
| 110 | 91 |
instance = self._get_instance(request.instance_name)
|
| 111 |
- while True:
|
|
| 112 |
- self.logger.debug("GetTree request: [{}]".format(request))
|
|
| 113 |
- # Returns next page_token once page_size directories is reached.
|
|
| 114 |
- # The page_token, is essentially an index into the directories/digests list.
|
|
| 115 |
- page_token = instance.get_tree(
|
|
| 116 |
- request, directories, digests)
|
|
| 117 |
- |
|
| 118 |
- response = remote_execution_pb2.GetTreeResponse()
|
|
| 119 |
- if not page_token:
|
|
| 120 |
- # get directories from last request to the end since no page_token
|
|
| 121 |
- response.directories = directories[start_index:]
|
|
| 122 |
- response.page_token = None
|
|
| 123 |
- # stop the generator no more directories
|
|
| 124 |
- return response
|
|
| 125 |
- else:
|
|
| 126 |
- # return from last request, to current request directories
|
|
| 127 |
- response.directories = directories[start_index:page_token]
|
|
| 128 |
- response.page_token = str(page_token)
|
|
| 129 |
- yield response
|
|
| 130 |
- |
|
| 131 |
- # create new request using returned page token, update start_index
|
|
| 132 |
- request = remote_execution_pb2.GetTreeRequest()
|
|
| 133 |
- request.page_size = 500
|
|
| 134 |
- request.page_token = page_token
|
|
| 135 |
- request.root_digest = digests[page_token]
|
|
| 136 |
- start_index = page_token
|
|
| 92 |
+ response = instance.get_tree(request)
|
|
| 93 |
+ |
|
| 94 |
+ return response
|
|
| 137 | 95 |
|
| 138 | 96 |
except InvalidArgumentError as e:
|
| 139 |
- self.logger.error(e)
|
|
| 97 |
+ self.__logger.error(e)
|
|
| 140 | 98 |
context.set_details(str(e))
|
| 141 | 99 |
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
| 142 | 100 |
|