[Notes] [Git][BuildStream/buildstream][richardmaw/builddir-sockets] 11 commits: _artifactcache/casserver.py: Implement Capabilities service

[richardmaw-codethink <gitlab mg gitlab com>]

Title: GitLab

richardmaw-codethink pushed to branch richardmaw/builddir-sockets at BuildStream / buildstream

Commits:

• 47f3064a
  by Jürg Billeter at 2018-09-10T15:43:25Z

  _artifactcache/casserver.py: Implement Capabilities service
  

• 1a7fb3cb
  by Jürg Billeter at 2018-09-10T15:43:25Z

  _artifactcache/casserver.py: Implement BatchReadBlobs
  
  Fixes #632.
  

• b3ffcdc8
  by Jürg Billeter at 2018-09-10T16:07:30Z

  Merge branch 'juerg/cas-batch' into 'master'
  
  _artifactcache/casserver.py: Implement BatchReadBlobs
  
  Closes #632
  
  See merge request BuildStream/buildstream!785

• 6f925bcb
  by Javier Jardón at 2018-09-13T07:38:48Z

  source/install_source.rst: pip plugin depends on host pip
  

• c6155f8d
  by Javier Jardón at 2018-09-13T08:11:54Z

  Merge branch 'jjardon/pip_dependency' into 'master'
  
  source/install_source.rst: pip plugin depends on host pip
  
  See merge request BuildStream/buildstream!791

• 19838a07
  by Chandan Singh at 2018-09-13T10:58:38Z

  README.rst: Add status badges for PyPI release and Python versions
  
  The first badge will work fine right away while the second badge will
  show "not found" until a release is made after merging this branch:
  https://gitlab.com/BuildStream/buildstream/merge_requests/718.
  

• 3b81d451
  by Chandan Singh at 2018-09-13T12:27:59Z

  Merge branch 'chandan/pypi-badge' into 'master'
  
  README.rst: Add status badges for PyPI release and Python versions
  
  See merge request BuildStream/buildstream!719

• 55956762
  by Richard Maw at 2018-09-13T16:50:33Z

  element: Remove __cached field
  
  This can get out of sync with other two cache states,
  and we can do without it.
  

• fc7f83ac
  by richardmaw-codethink at 2018-09-13T17:14:31Z

  Merge branch 'richardmaw/element-cache-state-simplify' into 'master'
  
  Simplify element state by removing `__cached`
  
  See merge request BuildStream/buildstream!784

• 233a7d83
  by Richard Maw at 2018-09-14T08:53:14Z

  Handle sockets when copying trees into artifacts
  
  We can't include a socket in a CAS tree, but it's mostly meaningless to do so
  since there can't possibly be a process serving it.
  

• f06f234a
  by Richard Maw at 2018-09-14T08:53:14Z

  tests: cover builds that make sockets
  

9 changed files:

• README.rst
• buildstream/_artifactcache/cascache.py
• buildstream/_artifactcache/casserver.py
• buildstream/element.py
• buildstream/utils.py
• doc/source/install_source.rst
• + tests/integration/project/elements/sockets/make-builddir-socket.bst
• + tests/integration/project/elements/sockets/make-install-root-socket.bst
• + tests/integration/sockets.py

Changes:

README.rst

---

...	...	@@ -13,6 +13,9 @@ About
13	13	.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/coverage.svg?job=coverage
14	14	:target: https://gitlab.com/BuildStream/buildstream/commits/master
15	15
	16	+.. image:: https://img.shields.io/pypi/v/BuildStream.svg
	17	+ :target: https://pypi.org/project/BuildStream
	18	+
16	19
17	20	What is BuildStream?
18	21	====================

buildstream/_artifactcache/cascache.py

---

...	...	@@ -684,6 +684,9 @@ class CASCache(ArtifactCache):
684	684	symlinknode = directory.symlinks.add()
685	685	symlinknode.name = name
686	686	symlinknode.target = os.readlink(full_path)
	687	+ elif stat.S_ISSOCK(mode):
	688	+ # The process serving the socket can't be cached anyway
	689	+ pass
687	690	else:
688	691	raise ArtifactError("Unsupported file type for {}".format(full_path))
689	692

buildstream/_artifactcache/casserver.py

---

...	...	@@ -38,6 +38,10 @@ from .._context import Context
38	38	from .cascache import CASCache
39	39
40	40
	41	+# The default limit for gRPC messages is 4 MiB
	42	+_MAX_BATCH_TOTAL_SIZE_BYTES = 4 * 1024 * 1024
	43	+
	44	+
41	45	# Trying to push an artifact that is too large
42	46	class ArtifactTooLargeException(Exception):
43	47	pass
...	...	@@ -67,6 +71,9 @@ def create_server(repo, *, enable_push):
67	71	remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(
68	72	_ContentAddressableStorageServicer(artifactcache), server)
69	73
	74	+ remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(
	75	+ _CapabilitiesServicer(), server)
	76	+
70	77	buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(
71	78	_ReferenceStorageServicer(artifactcache, enable_push=enable_push), server)
72	79
...	...	@@ -229,6 +236,48 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
229	236	d.size_bytes = digest.size_bytes
230	237	return response
231	238
	239	+ def BatchReadBlobs(self, request, context):
	240	+ response = remote_execution_pb2.BatchReadBlobsResponse()
	241	+ batch_size = 0
	242	+
	243	+ for digest in request.digests:
	244	+ batch_size += digest.size_bytes
	245	+ if batch_size > _MAX_BATCH_TOTAL_SIZE_BYTES:
	246	+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
	247	+ return response
	248	+
	249	+ blob_response = response.responses.add()
	250	+ blob_response.digest.hash = digest.hash
	251	+ blob_response.digest.size_bytes = digest.size_bytes
	252	+ try:
	253	+ with open(self.cas.objpath(digest), 'rb') as f:
	254	+ if os.fstat(f.fileno()).st_size != digest.size_bytes:
	255	+ blob_response.status.code = grpc.StatusCode.NOT_FOUND
	256	+ continue
	257	+
	258	+ blob_response.data = f.read(digest.size_bytes)
	259	+ except FileNotFoundError:
	260	+ blob_response.status.code = grpc.StatusCode.NOT_FOUND
	261	+
	262	+ return response
	263	+
	264	+
	265	+class _CapabilitiesServicer(remote_execution_pb2_grpc.CapabilitiesServicer):
	266	+ def GetCapabilities(self, request, context):
	267	+ response = remote_execution_pb2.ServerCapabilities()
	268	+
	269	+ cache_capabilities = response.cache_capabilities
	270	+ cache_capabilities.digest_function.append(remote_execution_pb2.SHA256)
	271	+ cache_capabilities.action_cache_update_capabilities.update_enabled = False
	272	+ cache_capabilities.max_batch_total_size_bytes = _MAX_BATCH_TOTAL_SIZE_BYTES
	273	+ cache_capabilities.symlink_absolute_path_strategy = remote_execution_pb2.CacheCapabilities.ALLOWED
	274	+
	275	+ response.deprecated_api_version.major = 2
	276	+ response.low_api_version.major = 2
	277	+ response.high_api_version.major = 2
	278	+
	279	+ return response
	280	+
232	281
233	282	class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
234	283	def __init__(self, cas, *, enable_push):

buildstream/element.py

---

...	...	@@ -200,7 +200,6 @@ class Element(Plugin):
200	200	self.__strict_cache_key = None # Our cached cache key for strict builds
201	201	self.__artifacts = artifacts # Artifact cache
202	202	self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state
203		- self.__cached = None # Whether we have a cached artifact
204	203	self.__strong_cached = None # Whether we have a cached artifact
205	204	self.__weak_cached = None # Whether we have a cached artifact
206	205	self.__assemble_scheduled = False # Element is scheduled to be assembled
...	...	@@ -1126,8 +1125,6 @@ class Element(Plugin):
1126	1125
1127	1126	# Query caches now that the weak and strict cache keys are available
1128	1127	key_for_cache_lookup = self.__strict_cache_key if context.get_strict() else self.__weak_cache_key
1129		- if not self.__cached:
1130		- self.__cached = self.__artifacts.contains(self, key_for_cache_lookup)
1131	1128	if not self.__strong_cached:
1132	1129	self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key)
1133	1130	if key_for_cache_lookup == self.__weak_cache_key:
...	...	@@ -2079,7 +2076,7 @@ class Element(Plugin):
2079	2076
2080	2077	def __is_cached(self, keystrength):
2081	2078	if keystrength is None:
2082		- return self.__cached
	2079	+ keystrength = _KeyStrength.STRONG if self._get_context().get_strict() else _KeyStrength.WEAK
2083	2080
2084	2081	return self.__strong_cached if keystrength == _KeyStrength.STRONG else self.__weak_cached
2085	2082

buildstream/utils.py

---

...	...	@@ -372,6 +372,8 @@ def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=Fa
372	372	Directories in `dest` are replaced with files from `src`,
373	373	unless the existing directory in `dest` is not empty in which
374	374	case the path will be reported in the return value.
	375	+
	376	+ UNIX domain socket files from `src` are ignored.
375	377	"""
376	378	presorted = False
377	379	if files is None:
...	...	@@ -414,6 +416,8 @@ def link_files(src, dest, *, files=None, ignore_missing=False, report_written=Fa
414	416
415	417	If a hardlink cannot be created due to crossing filesystems,
416	418	then the file will be copied instead.
	419	+
	420	+ UNIX domain socket files from `src` are ignored.
417	421	"""
418	422	presorted = False
419	423	if files is None:
...	...	@@ -841,6 +845,13 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
841	845	os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev)
842	846	os.chmod(destpath, file_stat.st_mode)
843	847
	848	+ elif stat.S_ISFIFO(mode):
	849	+ os.mkfifo(destpath, mode)
	850	+
	851	+ elif stat.S_ISSOCK(mode):
	852	+ # We can't duplicate the process serving the socket anyway
	853	+ pass
	854	+
844	855	else:
845	856	# Unsupported type.
846	857	raise UtilError('Cannot extract {} into staging-area. Unsupported type.'.format(srcpath))

doc/source/install_source.rst

---

...	...	@@ -29,6 +29,7 @@ The default plugins with extra host dependencies are:
29	29	* git
30	30	* ostree
31	31	* patch
	32	+* pip
32	33	* tar
33	34
34	35	If you intend to push built artifacts to a remote artifact server,

tests/integration/project/elements/sockets/make-builddir-socket.bst

---

	1	+kind: manual
	2	+
	3	+depends:
	4	+- filename: base.bst
	5	+ type: build
	6	+
	7	+config:
	8	+ build-commands:
	9	+ - |
	10	+ python3 -c '
	11	+ from socket import socket, AF_UNIX, SOCK_STREAM
	12	+ s = socket(AF_UNIX, SOCK_STREAM)
	13	+ s.bind("testsocket")
	14	+ '

tests/integration/project/elements/sockets/make-install-root-socket.bst

---

	1	+kind: manual
	2	+
	3	+depends:
	4	+- filename: base.bst
	5	+ type: build
	6	+
	7	+config:
	8	+ install-commands:
	9	+ - |
	10	+ python3 -c '
	11	+ from os.path import join
	12	+ from sys import argv
	13	+ from socket import socket, AF_UNIX, SOCK_STREAM
	14	+ s = socket(AF_UNIX, SOCK_STREAM)
	15	+ s.bind(join(argv[1], "testsocket"))
	16	+ ' %{install-root}

tests/integration/sockets.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from buildstream import _yaml
	5	+
	6	+from tests.testutils import cli_integration as cli
	7	+from tests.testutils.integration import assert_contains
	8	+
	9	+
	10	+pytestmark = pytest.mark.integration
	11	+
	12	+DATA_DIR = os.path.join(
	13	+ os.path.dirname(os.path.realpath(__file__)),
	14	+ "project"
	15	+)
	16	+
	17	+
	18	+@pytest.mark.datafiles(DATA_DIR)
	19	+def test_builddir_socket_ignored(cli, tmpdir, datafiles):
	20	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	21	+ element_name = 'sockets/make-builddir-socket.bst'
	22	+
	23	+ result = cli.run(project=project, args=['build', element_name])
	24	+ assert result.exit_code == 0
	25	+
	26	+
	27	+@pytest.mark.datafiles(DATA_DIR)
	28	+def test_install_root_socket_ignored(cli, tmpdir, datafiles):
	29	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	30	+ element_name = 'sockets/make-install-root-socket.bst'
	31	+
	32	+ result = cli.run(project=project, args=['build', element_name])
	33	+ assert result.exit_code == 0

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.