[Notes] [Git][BuildStream/buildstream][raoul/870-root-cache-dir] 3 commits: cachedir: add new dir option that's default root to other dirs

[Raoul Hidalgo Charman <gitlab mg gitlab com>]

Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/870-root-cache-dir at BuildStream / buildstream

Commits:

• 962b6c40
  by Raoul Hidalgo Charman at 2019-02-13T10:09:20Z

  cachedir: add new dir option that's default root to other dirs
  
  Makes artifactdir and builddir obsolete.
  
  Fixes #870
  

• 480d3a43
  by Raoul Hidalgo Charman at 2019-02-13T11:37:56Z

  context: remove artifactdir
  
  Will check and move old artifact directory if it exists, and create
  symlink linking old directory to new.
  

• 80377e94
  by Raoul Hidalgo Charman at 2019-02-13T13:53:49Z

  CASQuota: Move cache check methods into new Class
  
  This sits in Context allowing artifact cache to check the cas quota
  while not being used for CASServer.
  
  A lot of code that checks cache quota has been touched.
  
  Part of #870
  

28 changed files:

• buildstream/_artifactcache.py
• buildstream/_cas/__init__.py
• buildstream/_cas/cascache.py
• buildstream/_context.py
• buildstream/_frontend/status.py
• buildstream/_frontend/widget.py
• buildstream/_scheduler/jobs/cachesizejob.py
• buildstream/_scheduler/jobs/cleanupjob.py
• buildstream/data/userconfig.yaml
• buildstream/element.py
• buildstream/plugintestutils/runcli.py
• conftest.py
• doc/bst2html.py
• doc/sessions/running-commands.run
• tests/artifactcache/cache_size.py
• tests/artifactcache/expiry.py
• tests/artifactcache/junctions.py
• tests/artifactcache/pull.py
• tests/artifactcache/push.py
• tests/frontend/pull.py
• tests/integration/build-tree.py
• tests/integration/cachedfail.py
• tests/integration/messages.py
• tests/integration/pullbuildtrees.py
• tests/integration/source-determinism.py
• tests/internals/context.py
• tests/internals/pluginloading.py
• tests/testutils/artifactshare.py

Changes:

buildstream/_artifactcache.py

---

...	...	@@ -23,7 +23,7 @@ import string
23	23	from collections.abc import Mapping
24	24
25	25	from .types import _KeyStrength
26		-from ._exceptions import ArtifactError, CASError, LoadError, LoadErrorReason
	26	+from ._exceptions import ArtifactError, CASError
27	27	from ._message import Message, MessageType
28	28	from . import utils
29	29	from . import _yaml
...	...	@@ -46,39 +46,6 @@ class ArtifactCacheSpec(CASRemoteSpec):
46	46	pass
47	47
48	48
49		-# ArtifactCacheUsage
50		-#
51		-# A simple object to report the current artifact cache
52		-# usage details.
53		-#
54		-# Note that this uses the user configured cache quota
55		-# rather than the internal quota with protective headroom
56		-# removed, to provide a more sensible value to display to
57		-# the user.
58		-#
59		-# Args:
60		-# artifacts (ArtifactCache): The artifact cache to get the status of
61		-#
62		-class ArtifactCacheUsage():
63		-
64		- def __init__(self, artifacts):
65		- context = artifacts.context
66		- self.quota_config = context.config_cache_quota # Configured quota
67		- self.quota_size = artifacts._cache_quota_original # Resolved cache quota in bytes
68		- self.used_size = artifacts.get_cache_size() # Size used by artifacts in bytes
69		- self.used_percent = 0 # Percentage of the quota used
70		- if self.quota_size is not None:
71		- self.used_percent = int(self.used_size * 100 / self.quota_size)
72		-
73		- # Formattable into a human readable string
74		- #
75		- def __str__(self):
76		- return "{} / {} ({}%)" \
77		- .format(utils._pretty_size(self.used_size, dec_places=1),
78		- self.quota_config,
79		- self.used_percent)
80		-
81		-
82	49	# An ArtifactCache manages artifacts.
83	50	#
84	51	# Args:
...	...	@@ -87,19 +54,16 @@ class ArtifactCacheUsage():
87	54	class ArtifactCache():
88	55	def __init__(self, context):
89	56	self.context = context
90		- self.extractdir = os.path.join(context.artifactdir, 'extract')
	57	+ self.extractdir = context.extractdir
91	58
92	59	self.cas = context.get_cascache()
	60	+ self.casquota = context.get_casquota()
93	61
94	62	self.global_remote_specs = []
95	63	self.project_remote_specs = {}
96	64
97	65	self._required_elements = set() # The elements required for this session
98		- self._cache_size = None # The current cache size, sometimes it's an estimate
99		- self._cache_quota = None # The cache quota
100		- self._cache_quota_original = None # The cache quota as specified by the user, in bytes
101		- self._cache_quota_headroom = None # The headroom in bytes before reaching the quota or full disk
102		- self._cache_lower_threshold = None # The target cache size for a cleanup
	66	+
103	67	self._remotes_setup = False # Check to prevent double-setup of remotes
104	68
105	69	# Per-project list of _CASRemote instances.
...	...	@@ -110,8 +74,6 @@ class ArtifactCache():
110	74
111	75	os.makedirs(self.extractdir, exist_ok=True)
112	76
113		- self._calculate_cache_quota()
114		-
115	77	# get_artifact_fullname()
116	78	#
117	79	# Generate a full name for an artifact, including the
...	...	@@ -266,7 +228,7 @@ class ArtifactCache():
266	228	space_saved = 0
267	229
268	230	# Start off with an announcement with as much info as possible
269		- volume_size, volume_avail = self._get_cache_volume_size()
	231	+ volume_size, volume_avail = self.casquota._get_cache_volume_size()
270	232	self._message(MessageType.STATUS, "Starting cache cleanup",
271	233	detail=("Elements required by the current build plan: {}\n" +
272	234	"User specified quota: {} ({})\n" +
...	...	@@ -274,8 +236,8 @@ class ArtifactCache():
274	236	"Cache volume: {} total, {} available")
275	237	.format(len(self._required_elements),
276	238	context.config_cache_quota,
277		- utils._pretty_size(self._cache_quota_original, dec_places=2),
278		- utils._pretty_size(self.get_cache_size(), dec_places=2),
	239	+ utils._pretty_size(self.casquota._cache_quota, dec_places=2),
	240	+ utils._pretty_size(self.casquota.get_cache_size(), dec_places=2),
279	241	utils._pretty_size(volume_size, dec_places=2),
280	242	utils._pretty_size(volume_avail, dec_places=2)))
281	243
...	...	@@ -292,9 +254,10 @@ class ArtifactCache():
292	254	])
293	255
294	256	# Do a real computation of the cache size once, just in case
295		- self.compute_cache_size()
	257	+ usage = self.casquota.compute_cache_size()
	258	+ self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
296	259
297		- while self.get_cache_size() >= self._cache_lower_threshold:
	260	+ while self.casquota.get_cache_size() >= self.casquota._cache_lower_threshold:
298	261	try:
299	262	to_remove = artifacts.pop(0)
300	263	except IndexError:
...	...	@@ -311,7 +274,7 @@ class ArtifactCache():
311	274	"Please increase the cache-quota in {} and/or make more disk space."
312	275	.format(removed_ref_count,
313	276	utils._pretty_size(space_saved, dec_places=2),
314		- utils._pretty_size(self.get_cache_size(), dec_places=2),
	277	+ utils._pretty_size(self.casquota.get_cache_size(), dec_places=2),
315	278	len(self._required_elements),
316	279	(context.config_origin or default_conf)))
317	280
...	...	@@ -337,7 +300,7 @@ class ArtifactCache():
337	300	to_remove))
338	301
339	302	# Remove the size from the removed size
340		- self.set_cache_size(self._cache_size - size)
	303	+ self.casquota.set_cache_size(self.casquota._cache_size - size)
341	304
342	305	# User callback
343	306	#
...	...	@@ -353,29 +316,12 @@ class ArtifactCache():
353	316	"Cache usage is now: {}")
354	317	.format(removed_ref_count,
355	318	utils._pretty_size(space_saved, dec_places=2),
356		- utils._pretty_size(self.get_cache_size(), dec_places=2)))
357		-
358		- return self.get_cache_size()
359		-
360		- # compute_cache_size()
361		- #
362		- # Computes the real artifact cache size by calling
363		- # the abstract calculate_cache_size() method.
364		- #
365		- # Returns:
366		- # (int): The size of the artifact cache.
367		- #
368		- def compute_cache_size(self):
369		- old_cache_size = self._cache_size
370		- new_cache_size = self.cas.calculate_cache_size()
371		-
372		- if old_cache_size != new_cache_size:
373		- self._cache_size = new_cache_size
	319	+ utils._pretty_size(self.casquota.get_cache_size(), dec_places=2)))
374	320
375		- usage = ArtifactCacheUsage(self)
376		- self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
	321	+ return self.casquota.get_cache_size()
377	322
378		- return self._cache_size
	323	+ def full(self):
	324	+ return self.casquota.full()
379	325
380	326	# add_artifact_size()
381	327	#
...	...	@@ -386,71 +332,10 @@ class ArtifactCache():
386	332	# artifact_size (int): The size to add.
387	333	#
388	334	def add_artifact_size(self, artifact_size):
389		- cache_size = self.get_cache_size()
	335	+ cache_size = self.casquota.get_cache_size()
390	336	cache_size += artifact_size
391	337
392		- self.set_cache_size(cache_size)
393		-
394		- # get_cache_size()
395		- #
396		- # Fetches the cached size of the cache, this is sometimes
397		- # an estimate and periodically adjusted to the real size
398		- # when a cache size calculation job runs.
399		- #
400		- # When it is an estimate, the value is either correct, or
401		- # it is greater than the actual cache size.
402		- #
403		- # Returns:
404		- # (int) An approximation of the artifact cache size, in bytes.
405		- #
406		- def get_cache_size(self):
407		-
408		- # If we don't currently have an estimate, figure out the real cache size.
409		- if self._cache_size is None:
410		- stored_size = self._read_cache_size()
411		- if stored_size is not None:
412		- self._cache_size = stored_size
413		- else:
414		- self.compute_cache_size()
415		-
416		- return self._cache_size
417		-
418		- # set_cache_size()
419		- #
420		- # Forcefully set the overall cache size.
421		- #
422		- # This is used to update the size in the main process after
423		- # having calculated in a cleanup or a cache size calculation job.
424		- #
425		- # Args:
426		- # cache_size (int): The size to set.
427		- #
428		- def set_cache_size(self, cache_size):
429		-
430		- assert cache_size is not None
431		-
432		- self._cache_size = cache_size
433		- self._write_cache_size(self._cache_size)
434		-
435		- # full()
436		- #
437		- # Checks if the artifact cache is full, either
438		- # because the user configured quota has been exceeded
439		- # or because the underlying disk is almost full.
440		- #
441		- # Returns:
442		- # (bool): True if the artifact cache is full
443		- #
444		- def full(self):
445		-
446		- if self.get_cache_size() > self._cache_quota:
447		- return True
448		-
449		- _, volume_avail = self._get_cache_volume_size()
450		- if volume_avail < self._cache_quota_headroom:
451		- return True
452		-
453		- return False
	338	+ self.casquota.set_cache_size(cache_size)
454	339
455	340	# preflight():
456	341	#
...	...	@@ -896,142 +781,6 @@ class ArtifactCache():
896	781	with self.context.timed_activity("Initializing remote caches", silent_nested=True):
897	782	self.initialize_remotes(on_failure=remote_failed)
898	783
899		- # _write_cache_size()
900		- #
901		- # Writes the given size of the artifact to the cache's size file
902		- #
903		- # Args:
904		- # size (int): The size of the artifact cache to record
905		- #
906		- def _write_cache_size(self, size):
907		- assert isinstance(size, int)
908		- size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
909		- with utils.save_file_atomic(size_file_path, "w") as f:
910		- f.write(str(size))
911		-
912		- # _read_cache_size()
913		- #
914		- # Reads and returns the size of the artifact cache that's stored in the
915		- # cache's size file
916		- #
917		- # Returns:
918		- # (int): The size of the artifact cache, as recorded in the file
919		- #
920		- def _read_cache_size(self):
921		- size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
922		-
923		- if not os.path.exists(size_file_path):
924		- return None
925		-
926		- with open(size_file_path, "r") as f:
927		- size = f.read()
928		-
929		- try:
930		- num_size = int(size)
931		- except ValueError as e:
932		- raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format(
933		- size, size_file_path)) from e
934		-
935		- return num_size
936		-
937		- # _calculate_cache_quota()
938		- #
939		- # Calculates and sets the cache quota and lower threshold based on the
940		- # quota set in Context.
941		- # It checks that the quota is both a valid _expression_, and that there is
942		- # enough disk space to satisfy that quota
943		- #
944		- def _calculate_cache_quota(self):
945		- # Headroom intended to give BuildStream a bit of leeway.
946		- # This acts as the minimum size of cache_quota and also
947		- # is taken from the user requested cache_quota.
948		- #
949		- if 'BST_TEST_SUITE' in os.environ:
950		- self._cache_quota_headroom = 0
951		- else:
952		- self._cache_quota_headroom = 2e9
953		-
954		- try:
955		- cache_quota = utils._parse_size(self.context.config_cache_quota,
956		- self.context.artifactdir)
957		- except utils.UtilError as e:
958		- raise LoadError(LoadErrorReason.INVALID_DATA,
959		- "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
960		- "\nValid values are, for example: 800M 10G 1T 50%\n"
961		- .format(str(e))) from e
962		-
963		- total_size, available_space = self._get_cache_volume_size()
964		- cache_size = self.get_cache_size()
965		-
966		- # Ensure system has enough storage for the cache_quota
967		- #
968		- # If cache_quota is none, set it to the maximum it could possibly be.
969		- #
970		- # Also check that cache_quota is at least as large as our headroom.
971		- #
972		- if cache_quota is None: # Infinity, set to max system storage
973		- cache_quota = cache_size + available_space
974		- if cache_quota < self._cache_quota_headroom: # Check minimum
975		- raise LoadError(LoadErrorReason.INVALID_DATA,
976		- "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
977		- "BuildStream requires a minimum cache quota of 2G.")
978		- elif cache_quota > total_size:
979		- # A quota greater than the total disk size is certianly an error
980		- raise ArtifactError("Your system does not have enough available " +
981		- "space to support the cache quota specified.",
982		- detail=("You have specified a quota of {quota} total disk space.\n" +
983		- "The filesystem containing {local_cache_path} only " +
984		- "has {total_size} total disk space.")
985		- .format(
986		- quota=self.context.config_cache_quota,
987		- local_cache_path=self.context.artifactdir,
988		- total_size=utils._pretty_size(total_size)),
989		- reason='insufficient-storage-for-quota')
990		- elif cache_quota > cache_size + available_space:
991		- # The quota does not fit in the available space, this is a warning
992		- if '%' in self.context.config_cache_quota:
993		- available = (available_space / total_size) * 100
994		- available = '{}% of total disk space'.format(round(available, 1))
995		- else:
996		- available = utils._pretty_size(available_space)
997		-
998		- self._message(MessageType.WARN,
999		- "Your system does not have enough available " +
1000		- "space to support the cache quota specified.",
1001		- detail=("You have specified a quota of {quota} total disk space.\n" +
1002		- "The filesystem containing {local_cache_path} only " +
1003		- "has {available_size} available.")
1004		- .format(quota=self.context.config_cache_quota,
1005		- local_cache_path=self.context.artifactdir,
1006		- available_size=available))
1007		-
1008		- # Place a slight headroom (2e9 (2GB) on the cache_quota) into
1009		- # cache_quota to try and avoid exceptions.
1010		- #
1011		- # Of course, we might still end up running out during a build
1012		- # if we end up writing more than 2G, but hey, this stuff is
1013		- # already really fuzzy.
1014		- #
1015		- self._cache_quota_original = cache_quota
1016		- self._cache_quota = cache_quota - self._cache_quota_headroom
1017		- self._cache_lower_threshold = self._cache_quota / 2
1018		-
1019		- # _get_cache_volume_size()
1020		- #
1021		- # Get the available space and total space for the volume on
1022		- # which the artifact cache is located.
1023		- #
1024		- # Returns:
1025		- # (int): The total number of bytes on the volume
1026		- # (int): The number of available bytes on the volume
1027		- #
1028		- # NOTE: We use this stub to allow the test cases
1029		- # to override what an artifact cache thinks
1030		- # about it's disk size and available bytes.
1031		- #
1032		- def _get_cache_volume_size(self):
1033		- return utils._get_volume_size(self.context.artifactdir)
1034		-
1035	784
1036	785	# _configured_remote_artifact_cache_specs():
1037	786	#

buildstream/_cas/__init__.py

---

...	...	@@ -17,5 +17,5 @@
17	17	# Authors:
18	18	# Tristan Van Berkom <tristan vanberkom codethink co uk>
19	19
20		-from .cascache import CASCache
	20	+from .cascache import CASCache, CASQuota, CASCacheUsage
21	21	from .casremote import CASRemote, CASRemoteSpec

buildstream/_cas/cascache.py

---

...	...	@@ -31,17 +31,53 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
31	31	from .._protos.buildstream.v2 import buildstream_pb2
32	32
33	33	from .. import utils
34		-from .._exceptions import CASCacheError
	34	+from .._exceptions import CASCacheError, LoadError, LoadErrorReason
	35	+from .._message import Message, MessageType
35	36
36	37	from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate
37	38
38	39	_BUFFER_SIZE = 65536
39	40
40	41
	42	+CACHE_SIZE_FILE = "cache_size"
	43	+
	44	+
	45	+# CASCacheUsage
	46	+#
	47	+# A simple object to report the current CAS cache usage details.
	48	+#
	49	+# Note that this uses the user configured cache quota
	50	+# rather than the internal quota with protective headroom
	51	+# removed, to provide a more sensible value to display to
	52	+# the user.
	53	+#
	54	+# Args:
	55	+# cas (CASQuota): The CAS cache to get the status of
	56	+#
	57	+class CASCacheUsage():
	58	+
	59	+ def __init__(self, casquota):
	60	+ self.quota_config = casquota._config_cache_quota # Configured quota
	61	+ self.quota_size = casquota._cache_quota_original # Resolved cache quota in bytes
	62	+ self.used_size = casquota.get_cache_size() # Size used by artifacts in bytes
	63	+ self.used_percent = 0 # Percentage of the quota used
	64	+ if self.quota_size is not None:
	65	+ self.used_percent = int(self.used_size * 100 / self.quota_size)
	66	+
	67	+ # Formattable into a human readable string
	68	+ #
	69	+ def __str__(self):
	70	+ return "{} / {} ({}%)" \
	71	+ .format(utils._pretty_size(self.used_size, dec_places=1),
	72	+ self.quota_config,
	73	+ self.used_percent)
	74	+
	75	+
41	76	# A CASCache manages a CAS repository as specified in the Remote Execution API.
42	77	#
43	78	# Args:
44	79	# path (str): The root directory for the CAS repository
	80	+# cache_quota (int): User configured cache quota
45	81	#
46	82	class CASCache():
47	83
...	...	@@ -460,16 +496,6 @@ class CASCache():
460	496	except FileNotFoundError as e:
461	497	raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
462	498
463		- # calculate_cache_size()
464		- #
465		- # Return the real disk usage of the CAS cache.
466		- #
467		- # Returns:
468		- # (int): The size of the cache.
469		- #
470		- def calculate_cache_size(self):
471		- return utils._get_dir_size(self.casdir)
472		-
473	499	# list_refs():
474	500	#
475	501	# List refs in Least Recently Modified (LRM) order.
...	...	@@ -1018,6 +1044,240 @@ class CASCache():
1018	1044	batch.send()
1019	1045
1020	1046
	1047	+class CASQuota:
	1048	+ def __init__(self, context):
	1049	+ self.cas = context.get_cascache()
	1050	+ self.casdir = self.cas.casdir
	1051	+ self._config_cache_quota = context.config_cache_quota
	1052	+ self._cache_size = None # The current cache size, sometimes it's an estimate
	1053	+ self._cache_quota = None # The cache quota
	1054	+ self._cache_quota_original = None # The cache quota as specified by the user, in bytes
	1055	+ self._cache_quota_headroom = None
	1056	+ self._cache_lower_threshold = None # The target cache size for a cleanup
	1057	+ self.available_space = None
	1058	+
	1059	+ self._message = context.message
	1060	+
	1061	+ self._calculate_cache_quota()
	1062	+
	1063	+ # compute_cache_size()
	1064	+ #
	1065	+ # Computes the real artifact cache size by calling
	1066	+ # the abstract calculate_cache_size() method.
	1067	+ #
	1068	+ # Returns:
	1069	+ # (int): The size of the artifact cache.
	1070	+ #
	1071	+ def compute_cache_size(self):
	1072	+ old_cache_size = self._cache_size
	1073	+ new_cache_size = self.calculate_cache_size()
	1074	+
	1075	+ if old_cache_size != new_cache_size:
	1076	+ self._cache_size = new_cache_size
	1077	+
	1078	+ return self._cache_size
	1079	+
	1080	+ # calculate_cache_size()
	1081	+ #
	1082	+ # Return the real disk usage of the CAS cache.
	1083	+ #
	1084	+ # Returns:
	1085	+ # (int): The size of the cache.
	1086	+ #
	1087	+ def calculate_cache_size(self):
	1088	+ return utils._get_dir_size(self.casdir)
	1089	+
	1090	+ # get_cache_size()
	1091	+ #
	1092	+ # Fetches the cached size of the cache, this is sometimes
	1093	+ # an estimate and periodically adjusted to the real size
	1094	+ # when a cache size calculation job runs.
	1095	+ #
	1096	+ # When it is an estimate, the value is either correct, or
	1097	+ # it is greater than the actual cache size.
	1098	+ #
	1099	+ # Returns:
	1100	+ # (int) An approximation of the artifact cache size, in bytes.
	1101	+ #
	1102	+ def get_cache_size(self):
	1103	+
	1104	+ # If we don't currently have an estimate, figure out the real cache size.
	1105	+ if self._cache_size is None:
	1106	+ stored_size = self._read_cache_size()
	1107	+ if stored_size is not None:
	1108	+ self._cache_size = stored_size
	1109	+ else:
	1110	+ self._cache_size = self.compute_cache_size()
	1111	+
	1112	+ return self._cache_size
	1113	+
	1114	+ # set_cache_size()
	1115	+ #
	1116	+ # Forcefully set the overall cache size.
	1117	+ #
	1118	+ # This is used to update the size in the main process after
	1119	+ # having calculated in a cleanup or a cache size calculation job.
	1120	+ #
	1121	+ # Args:
	1122	+ # cache_size (int): The size to set.
	1123	+ #
	1124	+ def set_cache_size(self, cache_size):
	1125	+
	1126	+ assert cache_size is not None
	1127	+
	1128	+ self._cache_size = cache_size
	1129	+ self._write_cache_size(self._cache_size)
	1130	+
	1131	+ # full()
	1132	+ #
	1133	+ # Checks if the artifact cache is full, either
	1134	+ # because the user configured quota has been exceeded
	1135	+ # or because the underlying disk is almost full.
	1136	+ #
	1137	+ # Returns:
	1138	+ # (bool): True if the artifact cache is full
	1139	+ #
	1140	+ def full(self):
	1141	+
	1142	+ if self.get_cache_size() > self._cache_quota:
	1143	+ return True
	1144	+
	1145	+ _, volume_avail = self._get_cache_volume_size()
	1146	+ if volume_avail < self._cache_quota_headroom:
	1147	+ return True
	1148	+
	1149	+ return False
	1150	+
	1151	+ ################################################
	1152	+ # Local Private Methods #
	1153	+ ################################################
	1154	+
	1155	+ # _read_cache_size()
	1156	+ #
	1157	+ # Reads and returns the size of the artifact cache that's stored in the
	1158	+ # cache's size file
	1159	+ #
	1160	+ # Returns:
	1161	+ # (int): The size of the artifact cache, as recorded in the file
	1162	+ #
	1163	+ def _read_cache_size(self):
	1164	+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
	1165	+
	1166	+ if not os.path.exists(size_file_path):
	1167	+ return None
	1168	+
	1169	+ with open(size_file_path, "r") as f:
	1170	+ size = f.read()
	1171	+
	1172	+ try:
	1173	+ num_size = int(size)
	1174	+ except ValueError as e:
	1175	+ raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
	1176	+ size, size_file_path)) from e
	1177	+
	1178	+ return num_size
	1179	+
	1180	+ # _write_cache_size()
	1181	+ #
	1182	+ # Writes the given size of the artifact to the cache's size file
	1183	+ #
	1184	+ # Args:
	1185	+ # size (int): The size of the artifact cache to record
	1186	+ #
	1187	+ def _write_cache_size(self, size):
	1188	+ assert isinstance(size, int)
	1189	+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
	1190	+ with utils.save_file_atomic(size_file_path, "w") as f:
	1191	+ f.write(str(size))
	1192	+
	1193	+ # _get_cache_volume_size()
	1194	+ #
	1195	+ # Get the available space and total space for the volume on
	1196	+ # which the artifact cache is located.
	1197	+ #
	1198	+ # Returns:
	1199	+ # (int): The total number of bytes on the volume
	1200	+ # (int): The number of available bytes on the volume
	1201	+ #
	1202	+ # NOTE: We use this stub to allow the test cases
	1203	+ # to override what an artifact cache thinks
	1204	+ # about it's disk size and available bytes.
	1205	+ #
	1206	+ def _get_cache_volume_size(self):
	1207	+ return utils._get_volume_size(self.casdir)
	1208	+
	1209	+ # _calculate_cache_quota()
	1210	+ #
	1211	+ # Calculates and sets the cache quota and lower threshold based on the
	1212	+ # quota set in Context.
	1213	+ # It checks that the quota is both a valid _expression_, and that there is
	1214	+ # enough disk space to satisfy that quota
	1215	+ #
	1216	+ def _calculate_cache_quota(self):
	1217	+ # Headroom intended to give BuildStream a bit of leeway.
	1218	+ # This acts as the minimum size of cache_quota and also
	1219	+ # is taken from the user requested cache_quota.
	1220	+ #
	1221	+ if 'BST_TEST_SUITE' in os.environ:
	1222	+ self._cache_quota_headroom = 0
	1223	+ else:
	1224	+ self._cache_quota_headroom = 2e9
	1225	+
	1226	+ total_size, available_space = self._get_cache_volume_size()
	1227	+ cache_size = self.get_cache_size()
	1228	+ self.available_space = available_space
	1229	+
	1230	+ # Ensure system has enough storage for the cache_quota
	1231	+ #
	1232	+ # If cache_quota is none, set it to the maximum it could possibly be.
	1233	+ #
	1234	+ # Also check that cache_quota is at least as large as our headroom.
	1235	+ #
	1236	+ cache_quota = self._config_cache_quota
	1237	+ if cache_quota is None: # Infinity, set to max system storage
	1238	+ cache_quota = cache_size + available_space
	1239	+ if cache_quota < self._cache_quota_headroom: # Check minimum
	1240	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	1241	+ "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
	1242	+ "BuildStream requires a minimum cache quota of 2G.")
	1243	+ elif cache_quota > total_size:
	1244	+ # A quota greater than the total disk size is certianly an error
	1245	+ raise CASCacheError("Your system does not have enough available " +
	1246	+ "space to support the cache quota specified.",
	1247	+ detail=("You have specified a quota of {quota} total disk space.\n" +
	1248	+ "The filesystem containing {local_cache_path} only " +
	1249	+ "has {total_size} total disk space.")
	1250	+ .format(
	1251	+ quota=self._config_cache_quota,
	1252	+ local_cache_path=self.casdir,
	1253	+ total_size=utils._pretty_size(total_size)),
	1254	+ reason='insufficient-storage-for-quota')
	1255	+
	1256	+ elif cache_quota > cache_size + available_space:
	1257	+ self._message(Message(
	1258	+ None,
	1259	+ MessageType.WARN,
	1260	+ "Your system does not have enough available " +
	1261	+ "space to support the cache quota specified.",
	1262	+ detail=("You have specified a quota of {quota} total disk space.\n" +
	1263	+ "The filesystem containing {local_cache_path} only " +
	1264	+ "has {available_size} available.")
	1265	+ .format(quota=self._config_cache_quota,
	1266	+ local_cache_path=self.casdir,
	1267	+ available_size=self.available_space)))
	1268	+
	1269	+ # Place a slight headroom (2e9 (2GB) on the cache_quota) into
	1270	+ # cache_quota to try and avoid exceptions.
	1271	+ #
	1272	+ # Of course, we might still end up running out during a build
	1273	+ # if we end up writing more than 2G, but hey, this stuff is
	1274	+ # already really fuzzy.
	1275	+ #
	1276	+ self._cache_quota_original = cache_quota
	1277	+ self._cache_quota = cache_quota - self._cache_quota_headroom
	1278	+ self._cache_lower_threshold = self._cache_quota / 2
	1279	+
	1280	+
1021	1281	def _grouper(iterable, n):
1022	1282	while True:
1023	1283	try:

buildstream/_context.py

---

...	...	@@ -30,8 +30,8 @@ from . import _yaml
30	30	from ._exceptions import LoadError, LoadErrorReason, BstError
31	31	from ._message import Message, MessageType
32	32	from ._profile import Topics, profile_start, profile_end
33		-from ._artifactcache import ArtifactCache, ArtifactCacheUsage
34		-from ._cas import CASCache
	33	+from ._artifactcache import ArtifactCache
	34	+from ._cas import CASCache, CASQuota, CASCacheUsage
35	35	from ._workspaces import Workspaces, WorkspaceProjectCache
36	36	from .plugin import _plugin_lookup
37	37	from .sandbox import SandboxRemote
...	...	@@ -58,18 +58,27 @@ class Context():
58	58	# Filename indicating which configuration file was used, or None for the defaults
59	59	self.config_origin = None
60	60
	61	+ # The directory under which other directories are based
	62	+ self.cachedir = None
	63	+
61	64	# The directory where various sources are stored
62	65	self.sourcedir = None
63	66
64	67	# The directory where build sandboxes will be created
65	68	self.builddir = None
66	69
	70	+ # The directory for CAS
	71	+ self.casdir = None
	72	+
	73	+ # Extract directory
	74	+ self.extractdir = None
	75	+
	76	+ # The directory for temporary files
	77	+ self.tmpdir = None
	78	+
67	79	# Default root location for workspaces
68	80	self.workspacedir = None
69	81
70		- # The local binary artifact cache directory
71		- self.artifactdir = None
72		-
73	82	# The locations from which to push and pull prebuilt artifacts
74	83	self.artifact_cache_specs = None
75	84
...	...	@@ -143,6 +152,7 @@ class Context():
143	152	self._log_handle = None
144	153	self._log_filename = None
145	154	self._cascache = None
	155	+ self._casquota = None
146	156	self._directory = directory
147	157
148	158	# load()
...	...	@@ -180,13 +190,22 @@ class Context():
180	190	user_config = _yaml.load(config)
181	191	_yaml.composite(defaults, user_config)
182	192
	193	+ # Give obsoletion warnings
	194	+ if defaults.get('builddir'):
	195	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	196	+ "builddir is obsolete, use cachedir")
	197	+
	198	+ if defaults.get('artifactdir'):
	199	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	200	+ "artifactdir is obsolete")
	201	+
183	202	_yaml.node_validate(defaults, [
184		- 'sourcedir', 'builddir', 'artifactdir', 'logdir',
185		- 'scheduler', 'artifacts', 'logging', 'projects',
186		- 'cache', 'prompt', 'workspacedir', 'remote-execution'
	203	+ 'cachedir', 'sourcedir', 'builddir', 'logdir', 'scheduler',
	204	+ 'artifacts', 'logging', 'projects', 'cache', 'prompt',
	205	+ 'workspacedir', 'remote-execution',
187	206	])
188	207
189		- for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
	208	+ for directory in ['cachedir', 'sourcedir', 'logdir', 'workspacedir']:
190	209	# Allow the ~ tilde expansion and any environment variables in
191	210	# path specification in the config files.
192	211	#
...	...	@@ -196,14 +215,33 @@ class Context():
196	215	path = os.path.normpath(path)
197	216	setattr(self, directory, path)
198	217
	218	+ # add directories not set by users
	219	+ self.extractdir = os.path.join(self.cachedir, 'extract')
	220	+ self.tmpdir = os.path.join(self.cachedir, 'tmp')
	221	+ self.casdir = os.path.join(self.cachedir, 'cas')
	222	+ self.builddir = os.path.join(self.cachedir, 'build')
	223	+
	224	+ # Move old artifact cas to cas if it exists and create symlink
	225	+ old_casdir = os.path.join(self.cachedir, 'artifacts', 'cas')
	226	+ if os.path.exists(old_casdir) and not os.path.islink(old_casdir):
	227	+ os.rename(old_casdir, self.casdir)
	228	+ os.symlink(self.casdir, old_casdir)
	229	+
199	230	# Load quota configuration
200		- # We need to find the first existing directory in the path of
201		- # our artifactdir - the artifactdir may not have been created
202		- # yet.
	231	+ # We need to find the first existing directory in the path of our
	232	+ # cachedir - the cachedir may not have been created yet.
203	233	cache = _yaml.node_get(defaults, Mapping, 'cache')
204	234	_yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
205	235
206		- self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
	236	+ config_cache_quota = _yaml.node_get(cache, str, 'quota')
	237	+ try:
	238	+ self.config_cache_quota = utils._parse_size(config_cache_quota,
	239	+ self.casdir)
	240	+ except utils.UtilError as e:
	241	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	242	+ "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
	243	+ "\nValid values are, for example: 800M 10G 1T 50%\n"
	244	+ .format(str(e))) from e
207	245
208	246	# Load artifact share configuration
209	247	self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
...	...	@@ -275,15 +313,15 @@ class Context():
275	313
276	314	return self._artifactcache
277	315
278		- # get_artifact_cache_usage()
	316	+ # get_cache_usage()
279	317	#
280	318	# Fetches the current usage of the artifact cache
281	319	#
282	320	# Returns:
283		- # (ArtifactCacheUsage): The current status
	321	+ # (CASCacheUsage): The current status
284	322	#
285		- def get_artifact_cache_usage(self):
286		- return ArtifactCacheUsage(self.artifactcache)
	323	+ def get_cache_usage(self):
	324	+ return CASCacheUsage(self.get_casquota())
287	325
288	326	# add_project():
289	327	#
...	...	@@ -650,9 +688,14 @@ class Context():
650	688
651	689	def get_cascache(self):
652	690	if self._cascache is None:
653		- self._cascache = CASCache(self.artifactdir)
	691	+ self._cascache = CASCache(self.cachedir)
654	692	return self._cascache
655	693
	694	+ def get_casquota(self):
	695	+ if self._casquota is None:
	696	+ self._casquota = CASQuota(self)
	697	+ return self._casquota
	698	+
656	699
657	700	# _node_get_option_str()
658	701	#

buildstream/_frontend/status.py

---

...	...	@@ -404,7 +404,7 @@ class _StatusHeader():
404	404	#
405	405	# ~~~~~~ cache: 69% ~~~~~~
406	406	#
407		- usage = self._context.get_artifact_cache_usage()
	407	+ usage = self._context.get_cache_usage()
408	408	usage_percent = '{}%'.format(usage.used_percent)
409	409
410	410	size = 21

buildstream/_frontend/widget.py

---

...	...	@@ -486,7 +486,7 @@ class LogLine(Widget):
486	486	values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
487	487	values["Project"] = "{} ({})".format(project.name, project.directory)
488	488	values["Targets"] = ", ".join([t.name for t in stream.targets])
489		- values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage())
	489	+ values["Cache Usage"] = "{}".format(context.get_cache_usage())
490	490	text += self._format_values(values)
491	491
492	492	# User configurations
...	...	@@ -495,10 +495,10 @@ class LogLine(Widget):
495	495	values = OrderedDict()
496	496	values["Configuration File"] = \
497	497	"Default Configuration" if not context.config_origin else context.config_origin
	498	+ values["Cache directory"] = context.cachedir
498	499	values["Log Files"] = context.logdir
499	500	values["Source Mirrors"] = context.sourcedir
500	501	values["Build Area"] = context.builddir
501		- values["Artifact Cache"] = context.artifactdir
502	502	values["Strict Build Plan"] = "Yes" if context.get_strict() else "No"
503	503	values["Maximum Fetch Tasks"] = context.sched_fetchers
504	504	values["Maximum Build Tasks"] = context.sched_builders

buildstream/_scheduler/jobs/cachesizejob.py

---

...	...	@@ -25,14 +25,14 @@ class CacheSizeJob(Job):
25	25	self._complete_cb = complete_cb
26	26
27	27	context = self._scheduler.context
28		- self._artifacts = context.artifactcache
	28	+ self._casquota = context.get_casquota()
29	29
30	30	def child_process(self):
31		- return self._artifacts.compute_cache_size()
	31	+ return self._casquota.compute_cache_size()
32	32
33	33	def parent_complete(self, status, result):
34	34	if status == JobStatus.OK:
35		- self._artifacts.set_cache_size(result)
	35	+ self._casquota.set_cache_size(result)
36	36
37	37	if self._complete_cb:
38	38	self._complete_cb(status, result)

buildstream/_scheduler/jobs/cleanupjob.py

---

...	...	@@ -25,27 +25,27 @@ class CleanupJob(Job):
25	25	self._complete_cb = complete_cb
26	26
27	27	context = self._scheduler.context
	28	+ self._casquota = context.get_casquota()
28	29	self._artifacts = context.artifactcache
29	30
30	31	def child_process(self):
31	32	def progress():
32	33	self.send_message('update-cache-size',
33		- self._artifacts.get_cache_size())
	34	+ self._casquota.get_cache_size())
34	35	return self._artifacts.clean(progress)
35	36
36	37	def handle_message(self, message_type, message):
37		-
38	38	# Update the cache size in the main process as we go,
39	39	# this provides better feedback in the UI.
40	40	if message_type == 'update-cache-size':
41		- self._artifacts.set_cache_size(message)
	41	+ self._casquota.set_cache_size(message)
42	42	return True
43	43
44	44	return False
45	45
46	46	def parent_complete(self, status, result):
47	47	if status == JobStatus.OK:
48		- self._artifacts.set_cache_size(result)
	48	+ self._casquota.set_cache_size(result)
49	49
50	50	if self._complete_cb:
51	51	self._complete_cb(status, result)

buildstream/data/userconfig.yaml

---

...	...	@@ -13,11 +13,8 @@
13	13	# Location to store sources
14	14	sourcedir: ${XDG_CACHE_HOME}/buildstream/sources
15	15
16		-# Location to perform builds
17		-builddir: ${XDG_CACHE_HOME}/buildstream/build
18		-
19		-# Location to store local binary artifacts
20		-artifactdir: ${XDG_CACHE_HOME}/buildstream/artifacts
	16	+# Root location for other directories in the cache
	17	+cachedir: ${XDG_CACHE_HOME}/buildstream
21	18
22	19	# Location to store build logs
23	20	logdir: ${XDG_CACHE_HOME}/buildstream/logs

buildstream/element.py

---

...	...	@@ -1403,7 +1403,7 @@ class Element(Plugin):
1403	1403	# It's advantageous to have this temporary directory on
1404	1404	# the same file system as the rest of our cache.
1405	1405	with self.timed_activity("Staging sources", silent_nested=True), \
1406		- utils._tempdir(dir=context.artifactdir, prefix='staging-temp') as temp_staging_directory:
	1406	+ utils._tempdir(dir=context.tmpdir, prefix='staging-temp') as temp_staging_directory:
1407	1407
1408	1408	import_dir = temp_staging_directory
1409	1409

buildstream/plugintestutils/runcli.py

---

...	...	@@ -277,15 +277,13 @@ class Cli():
277	277	*, cache_dir=None):
278	278	# Read configuration to figure out where artifacts are stored
279	279	if not cache_dir:
280		- default = os.path.join(project, 'cache', 'artifacts')
281		-
282		- if self.config is not None:
283		- cache_dir = self.config.get('artifactdir', default)
284		- else:
285		- cache_dir = default
	280	+ cache_dir = os.path.join(project, 'cache')
286	281
287	282	cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
288	283
	284	+ # replace forward slashes
	285	+ element_name = element_name.replace('/', '-')
	286	+
289	287	cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
290	288	shutil.rmtree(cache_dir)
291	289
...	...	@@ -582,11 +580,21 @@ def cli_integration(tmpdir, integration_cache):
582	580	# We want to cache sources for integration tests more permanently,
583	581	# to avoid downloading the huge base-sdk repeatedly
584	582	fixture.configure({
	583	+ 'cachedir': integration_cache.root,
585	584	'sourcedir': integration_cache.sources,
586		- 'artifactdir': integration_cache.artifacts
587	585	})
588	586
589		- return fixture
	587	+ yield fixture
	588	+
	589	+ # remove following folders if necessary
	590	+ try:
	591	+ shutil.rmtree(os.path.join(integration_cache.root, 'build'))
	592	+ except FileNotFoundError:
	593	+ pass
	594	+ try:
	595	+ shutil.rmtree(os.path.join(integration_cache.root, 'tmp'))
	596	+ except FileNotFoundError:
	597	+ pass
590	598
591	599
592	600	@contextmanager
...	...	@@ -626,10 +634,8 @@ def configured(directory, config=None):
626	634
627	635	if not config.get('sourcedir', False):
628	636	config['sourcedir'] = os.path.join(directory, 'sources')
629		- if not config.get('builddir', False):
630		- config['builddir'] = os.path.join(directory, 'build')
631		- if not config.get('artifactdir', False):
632		- config['artifactdir'] = os.path.join(directory, 'artifacts')
	637	+ if not config.get('cachedir', False):
	638	+ config['cachedir'] = directory
633	639	if not config.get('logdir', False):
634	640	config['logdir'] = os.path.join(directory, 'logs')
635	641

conftest.py

---

...	...	@@ -53,16 +53,16 @@ def pytest_runtest_setup(item):
53	53	class IntegrationCache():
54	54
55	55	def __init__(self, cache):
56		- cache = os.path.abspath(cache)
	56	+ self.root = os.path.abspath(cache)
57	57	os.makedirs(cache, exist_ok=True)
58	58
59	59	# Use the same sources every time
60		- self.sources = os.path.join(cache, 'sources')
	60	+ self.sources = os.path.join(self.root, 'sources')
61	61
62	62	# Create a temp directory for the duration of the test for
63	63	# the artifacts directory
64	64	try:
65		- self.artifacts = tempfile.mkdtemp(dir=cache, prefix='artifacts-')
	65	+ self.artifacts = tempfile.mkdtemp(dir=self.root, prefix='artifacts-')
66	66	except OSError as e:
67	67	raise AssertionError("Unable to create test directory !") from e
68	68
...	...	@@ -87,6 +87,10 @@ def integration_cache(request):
87	87	shutil.rmtree(cache.artifacts)
88	88	except FileNotFoundError:
89	89	pass
	90	+ try:
	91	+ shutil.rmtree(os.path.join(cache.root, 'cas'))
	92	+ except FileNotFoundError:
	93	+ pass
90	94
91	95
92	96	#################################################

doc/bst2html.py

---

...	...	@@ -194,10 +194,9 @@ def workdir(source_cache=None):
194	194
195	195	bst_config_file = os.path.join(tempdir, 'buildstream.conf')
196	196	config = {
	197	+ 'cachedir': tempdir,
197	198	'sourcedir': source_cache,
198		- 'artifactdir': os.path.join(tempdir, 'artifacts'),
199	199	'logdir': os.path.join(tempdir, 'logs'),
200		- 'builddir': os.path.join(tempdir, 'build'),
201	200	}
202	201	_yaml.dump(config, bst_config_file)
203	202
...	...	@@ -411,12 +410,10 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
411	410	# Encode and save the output if that was asked for
412	411	output = _yaml.node_get(command, str, 'output', default_value=None)
413	412	if output is not None:
414		-
415	413	# Convert / Generate a nice <div>
416	414	converted = generate_html(command_out, directory, config_file,
417	415	source_cache, tempdir, palette,
418	416	command_str, command_fake_output is not None)
419		-
420	417	# Save it
421	418	filename = os.path.join(desc_dir, output)
422	419	filename = os.path.realpath(filename)

doc/sessions/running-commands.run

---

...	...	@@ -2,7 +2,7 @@
2	2	commands:
3	3	# Make it fetch first
4	4	- directory: ../examples/running-commands
5		- command: fetch hello.bst
	5	+ command: source fetch hello.bst
6	6
7	7	# Capture a show output
8	8	- directory: ../examples/running-commands

tests/artifactcache/cache_size.py

---

...	...	@@ -50,15 +50,15 @@ def test_cache_size_write(cli, tmpdir):
50	50	create_project(project_dir)
51	51
52	52	# Artifact cache must be in a known place
53		- artifactdir = os.path.join(project_dir, "artifacts")
54		- cli.configure({"artifactdir": artifactdir})
	53	+ casdir = os.path.join(project_dir, "cas")
	54	+ cli.configure({"cachedir": project_dir})
55	55
56	56	# Build, to populate the cache
57	57	res = cli.run(project=project_dir, args=["build", "test.bst"])
58	58	res.assert_success()
59	59
60	60	# Inspect the artifact cache
61		- sizefile = os.path.join(artifactdir, CACHE_SIZE_FILE)
	61	+ sizefile = os.path.join(casdir, CACHE_SIZE_FILE)
62	62	assert os.path.isfile(sizefile)
63	63	with open(sizefile, "r") as f:
64	64	size_data = f.read()
...	...	@@ -81,11 +81,11 @@ def test_quota_over_1024T(cli, tmpdir):
81	81	_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
82	82
83	83	volume_space_patch = mock.patch(
84		- "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
	84	+ "buildstream._cas.CASQuota._get_cache_volume_size",
85	85	autospec=True,
86	86	return_value=(1025 * TiB, 1025 * TiB)
87	87	)
88	88
89	89	with volume_space_patch:
90	90	result = cli.run(project, args=["build", "file.bst"])
91		- result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
	91	+ result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota')

tests/artifactcache/expiry.py

---

...	...	@@ -341,7 +341,7 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
341	341	("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
342	342
343	343	# Not enough space on disk even if you cleaned up
344		- ("11K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'),
	344	+ ("11K", ErrorDomain.CAS, 'insufficient-storage-for-quota'),
345	345
346	346	# Not enough space for these caches
347	347	("7K", 'warning', 'Your system does not have enough available'),
...	...	@@ -355,7 +355,7 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
355	355	cli.configure({
356	356	'cache': {
357	357	'quota': quota,
358		- }
	358	+ },
359	359	})
360	360
361	361	# We patch how we get space information
...	...	@@ -373,13 +373,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
373	373	total_space = 10000
374	374
375	375	volume_space_patch = mock.patch(
376		- "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
	376	+ "buildstream.utils._get_volume_size",
377	377	autospec=True,
378	378	return_value=(total_space, free_space),
379	379	)
380	380
381	381	cache_size_patch = mock.patch(
382		- "buildstream._artifactcache.ArtifactCache.get_cache_size",
	382	+ "buildstream._cas.CASQuota.get_cache_size",
383	383	autospec=True,
384	384	return_value=0,
385	385	)
...	...	@@ -417,7 +417,7 @@ def test_extract_expiry(cli, datafiles, tmpdir):
417	417	res.assert_success()
418	418
419	419	# Get a snapshot of the extracts in advance
420		- extractdir = os.path.join(project, 'cache', 'artifacts', 'extract', 'test', 'target')
	420	+ extractdir = os.path.join(project, 'cache', 'extract', 'test', 'target')
421	421	extracts = os.listdir(extractdir)
422	422	assert(len(extracts) == 1)
423	423	extract = os.path.join(extractdir, extracts[0])
...	...	@@ -436,7 +436,7 @@ def test_extract_expiry(cli, datafiles, tmpdir):
436	436	# Now we should have a directory for the cached target2.bst, which
437	437	# replaced target.bst in the cache, we should not have a directory
438	438	# for the target.bst
439		- refsdir = os.path.join(project, 'cache', 'artifacts', 'cas', 'refs', 'heads')
	439	+ refsdir = os.path.join(project, 'cache', 'cas', 'refs', 'heads')
440	440	refsdirtest = os.path.join(refsdir, 'test')
441	441	refsdirtarget = os.path.join(refsdirtest, 'target')
442	442	refsdirtarget2 = os.path.join(refsdirtest, 'target2')

tests/artifactcache/junctions.py

---

...	...	@@ -70,8 +70,8 @@ def test_push_pull(cli, tmpdir, datafiles):
70	70	# Now we've pushed, delete the user's local artifact cache
71	71	# directory and try to redownload it from the share
72	72	#
73		- artifacts = os.path.join(cli.directory, 'artifacts')
74		- shutil.rmtree(artifacts)
	73	+ cas = os.path.join(cli.directory, 'cas')
	74	+ shutil.rmtree(cas)
75	75
76	76	# Assert that nothing is cached locally anymore
77	77	state = cli.get_element_state(project, 'target.bst')

tests/artifactcache/pull.py

---

...	...	@@ -57,7 +57,7 @@ def test_pull(cli, tmpdir, datafiles):
57	57	# Set up an artifact cache.
58	58	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
59	59	# Configure artifact share
60		- artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	60	+ cache_dir = os.path.join(str(tmpdir), 'cache')
61	61	user_config_file = str(tmpdir.join('buildstream.conf'))
62	62	user_config = {
63	63	'scheduler': {
...	...	@@ -66,7 +66,8 @@ def test_pull(cli, tmpdir, datafiles):
66	66	'artifacts': {
67	67	'url': share.repo,
68	68	'push': True,
69		- }
	69	+ },
	70	+ 'cachedir': cache_dir
70	71	}
71	72
72	73	# Write down the user configuration file
...	...	@@ -93,7 +94,6 @@ def test_pull(cli, tmpdir, datafiles):
93	94	# Fake minimal context
94	95	context = Context()
95	96	context.load(config=user_config_file)
96		- context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
97	97	context.set_message_handler(message_handler)
98	98
99	99	# Load the project and CAS cache
...	...	@@ -111,7 +111,7 @@ def test_pull(cli, tmpdir, datafiles):
111	111	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
112	112	process = multiprocessing.Process(target=_queue_wrapper,
113	113	args=(_test_pull, queue, user_config_file, project_dir,
114		- artifact_dir, 'target.bst', element_key))
	114	+ cache_dir, 'target.bst', element_key))
115	115
116	116	try:
117	117	# Keep SIGINT blocked in the child process
...	...	@@ -128,12 +128,14 @@ def test_pull(cli, tmpdir, datafiles):
128	128	assert cas.contains(element, element_key)
129	129
130	130
131		-def _test_pull(user_config_file, project_dir, artifact_dir,
	131	+def _test_pull(user_config_file, project_dir, cache_dir,
132	132	element_name, element_key, queue):
133	133	# Fake minimal context
134	134	context = Context()
135	135	context.load(config=user_config_file)
136		- context.artifactdir = artifact_dir
	136	+ context.cachedir = cache_dir
	137	+ context.casdir = os.path.join(cache_dir, 'cas')
	138	+ context.tmpdir = os.path.join(cache_dir, 'tmp')
137	139	context.set_message_handler(message_handler)
138	140
139	141	# Load the project manually
...	...	@@ -166,7 +168,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
166	168	# Set up an artifact cache.
167	169	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
168	170	# Configure artifact share
169		- artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	171	+ rootcache_dir = os.path.join(str(tmpdir), 'cache')
170	172	user_config_file = str(tmpdir.join('buildstream.conf'))
171	173	user_config = {
172	174	'scheduler': {
...	...	@@ -175,7 +177,8 @@ def test_pull_tree(cli, tmpdir, datafiles):
175	177	'artifacts': {
176	178	'url': share.repo,
177	179	'push': True,
178		- }
	180	+ },
	181	+ 'cachedir': rootcache_dir
179	182	}
180	183
181	184	# Write down the user configuration file
...	...	@@ -196,7 +199,6 @@ def test_pull_tree(cli, tmpdir, datafiles):
196	199	# Fake minimal context
197	200	context = Context()
198	201	context.load(config=user_config_file)
199		- context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
200	202	context.set_message_handler(message_handler)
201	203
202	204	# Load the project and CAS cache
...	...	@@ -219,7 +221,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
219	221	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
220	222	process = multiprocessing.Process(target=_queue_wrapper,
221	223	args=(_test_push_tree, queue, user_config_file, project_dir,
222		- artifact_dir, artifact_digest))
	224	+ artifact_digest))
223	225
224	226	try:
225	227	# Keep SIGINT blocked in the child process
...	...	@@ -247,7 +249,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
247	249	# Use subprocess to avoid creation of gRPC threads in main BuildStream process
248	250	process = multiprocessing.Process(target=_queue_wrapper,
249	251	args=(_test_pull_tree, queue, user_config_file, project_dir,
250		- artifact_dir, tree_digest))
	252	+ tree_digest))
251	253
252	254	try:
253	255	# Keep SIGINT blocked in the child process
...	...	@@ -269,11 +271,10 @@ def test_pull_tree(cli, tmpdir, datafiles):
269	271	assert os.path.exists(cas.objpath(directory_digest))
270	272
271	273
272		-def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
	274	+def _test_push_tree(user_config_file, project_dir, artifact_digest, queue):
273	275	# Fake minimal context
274	276	context = Context()
275	277	context.load(config=user_config_file)
276		- context.artifactdir = artifact_dir
277	278	context.set_message_handler(message_handler)
278	279
279	280	# Load the project manually
...	...	@@ -305,11 +306,10 @@ def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest
305	306	queue.put("No remote configured")
306	307
307	308
308		-def _test_pull_tree(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
	309	+def _test_pull_tree(user_config_file, project_dir, artifact_digest, queue):
309	310	# Fake minimal context
310	311	context = Context()
311	312	context.load(config=user_config_file)
312		- context.artifactdir = artifact_dir
313	313	context.set_message_handler(message_handler)
314	314
315	315	# Load the project manually

tests/artifactcache/push.py

---

...	...	@@ -51,7 +51,7 @@ def test_push(cli, tmpdir, datafiles):
51	51	# Set up an artifact cache.
52	52	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
53	53	# Configure artifact share
54		- artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	54	+ rootcache_dir = os.path.join(str(tmpdir), 'cache')
55	55	user_config_file = str(tmpdir.join('buildstream.conf'))
56	56	user_config = {
57	57	'scheduler': {
...	...	@@ -60,7 +60,8 @@ def test_push(cli, tmpdir, datafiles):
60	60	'artifacts': {
61	61	'url': share.repo,
62	62	'push': True,
63		- }
	63	+ },
	64	+ 'cachedir': rootcache_dir
64	65	}
65	66
66	67	# Write down the user configuration file
...	...	@@ -69,7 +70,6 @@ def test_push(cli, tmpdir, datafiles):
69	70	# Fake minimal context
70	71	context = Context()
71	72	context.load(config=user_config_file)
72		- context.artifactdir = artifact_dir
73	73	context.set_message_handler(message_handler)
74	74
75	75	# Load the project manually
...	...	@@ -89,7 +89,7 @@ def test_push(cli, tmpdir, datafiles):
89	89	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
90	90	process = multiprocessing.Process(target=_queue_wrapper,
91	91	args=(_test_push, queue, user_config_file, project_dir,
92		- artifact_dir, 'target.bst', element_key))
	92	+ 'target.bst', element_key))
93	93
94	94	try:
95	95	# Keep SIGINT blocked in the child process
...	...	@@ -106,12 +106,10 @@ def test_push(cli, tmpdir, datafiles):
106	106	assert share.has_artifact('test', 'target.bst', element_key)
107	107
108	108
109		-def _test_push(user_config_file, project_dir, artifact_dir,
110		- element_name, element_key, queue):
	109	+def _test_push(user_config_file, project_dir, element_name, element_key, queue):
111	110	# Fake minimal context
112	111	context = Context()
113	112	context.load(config=user_config_file)
114		- context.artifactdir = artifact_dir
115	113	context.set_message_handler(message_handler)
116	114
117	115	# Load the project manually
...	...	@@ -152,7 +150,7 @@ def test_push_directory(cli, tmpdir, datafiles):
152	150	# Set up an artifact cache.
153	151	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
154	152	# Configure artifact share
155		- artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	153	+ rootcache_dir = os.path.join(str(tmpdir), 'cache')
156	154	user_config_file = str(tmpdir.join('buildstream.conf'))
157	155	user_config = {
158	156	'scheduler': {
...	...	@@ -161,7 +159,8 @@ def test_push_directory(cli, tmpdir, datafiles):
161	159	'artifacts': {
162	160	'url': share.repo,
163	161	'push': True,
164		- }
	162	+ },
	163	+ 'cachedir': rootcache_dir
165	164	}
166	165
167	166	# Write down the user configuration file
...	...	@@ -170,7 +169,6 @@ def test_push_directory(cli, tmpdir, datafiles):
170	169	# Fake minimal context
171	170	context = Context()
172	171	context.load(config=user_config_file)
173		- context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
174	172	context.set_message_handler(message_handler)
175	173
176	174	# Load the project and CAS cache
...	...	@@ -198,7 +196,7 @@ def test_push_directory(cli, tmpdir, datafiles):
198	196	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
199	197	process = multiprocessing.Process(target=_queue_wrapper,
200	198	args=(_test_push_directory, queue, user_config_file,
201		- project_dir, artifact_dir, artifact_digest))
	199	+ project_dir, artifact_digest))
202	200
203	201	try:
204	202	# Keep SIGINT blocked in the child process
...	...	@@ -216,11 +214,10 @@ def test_push_directory(cli, tmpdir, datafiles):
216	214	assert share.has_object(artifact_digest)
217	215
218	216
219		-def _test_push_directory(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
	217	+def _test_push_directory(user_config_file, project_dir, artifact_digest, queue):
220	218	# Fake minimal context
221	219	context = Context()
222	220	context.load(config=user_config_file)
223		- context.artifactdir = artifact_dir
224	221	context.set_message_handler(message_handler)
225	222
226	223	# Load the project manually
...	...	@@ -254,6 +251,7 @@ def test_push_message(cli, tmpdir, datafiles):
254	251	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
255	252	# Configure artifact share
256	253	artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	254	+ rootcache_dir = os.path.join(str(tmpdir), 'cache')
257	255	user_config_file = str(tmpdir.join('buildstream.conf'))
258	256	user_config = {
259	257	'scheduler': {
...	...	@@ -262,7 +260,8 @@ def test_push_message(cli, tmpdir, datafiles):
262	260	'artifacts': {
263	261	'url': share.repo,
264	262	'push': True,
265		- }
	263	+ },
	264	+ 'cachedir': rootcache_dir
266	265	}
267	266
268	267	# Write down the user configuration file
...	...	@@ -273,7 +272,7 @@ def test_push_message(cli, tmpdir, datafiles):
273	272	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
274	273	process = multiprocessing.Process(target=_queue_wrapper,
275	274	args=(_test_push_message, queue, user_config_file,
276		- project_dir, artifact_dir))
	275	+ project_dir))
277	276
278	277	try:
279	278	# Keep SIGINT blocked in the child process
...	...	@@ -292,11 +291,10 @@ def test_push_message(cli, tmpdir, datafiles):
292	291	assert share.has_object(message_digest)
293	292
294	293
295		-def _test_push_message(user_config_file, project_dir, artifact_dir, queue):
	294	+def _test_push_message(user_config_file, project_dir, queue):
296	295	# Fake minimal context
297	296	context = Context()
298	297	context.load(config=user_config_file)
299		- context.artifactdir = artifact_dir
300	298	context.set_message_handler(message_handler)
301	299
302	300	# Load the project manually

tests/frontend/pull.py

---

...	...	@@ -63,8 +63,8 @@ def test_push_pull_all(cli, tmpdir, datafiles):
63	63	# Now we've pushed, delete the user's local artifact cache
64	64	# directory and try to redownload it from the share
65	65	#
66		- artifacts = os.path.join(cli.directory, 'artifacts')
67		- shutil.rmtree(artifacts)
	66	+ cas = os.path.join(cli.directory, 'cas')
	67	+ shutil.rmtree(cas)
68	68
69	69	# Assert that nothing is cached locally anymore
70	70	states = cli.get_element_states(project, all_elements)
...	...	@@ -113,7 +113,7 @@ def test_push_pull_default_targets(cli, tmpdir, datafiles):
113	113	# Now we've pushed, delete the user's local artifact cache
114	114	# directory and try to redownload it from the share
115	115	#
116		- artifacts = os.path.join(cli.directory, 'artifacts')
	116	+ artifacts = os.path.join(cli.directory, 'cas')
117	117	shutil.rmtree(artifacts)
118	118
119	119	# Assert that nothing is cached locally anymore
...	...	@@ -155,8 +155,8 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles):
155	155	assert_shared(cli, share2, project, 'target.bst')
156	156
157	157	# Delete the user's local artifact cache.
158		- artifacts = os.path.join(cli.directory, 'artifacts')
159		- shutil.rmtree(artifacts)
	158	+ cas = os.path.join(cli.directory, 'cas')
	159	+ shutil.rmtree(cas)
160	160
161	161	# Assert that the element is not cached anymore.
162	162	assert cli.get_element_state(project, 'target.bst') != 'cached'
...	...	@@ -209,8 +209,8 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles):
209	209	# Now we've pushed, delete the user's local artifact cache
210	210	# directory and try to redownload it from the good_share.
211	211	#
212		- artifacts = os.path.join(cli.directory, 'artifacts')
213		- shutil.rmtree(artifacts)
	212	+ cas = os.path.join(cli.directory, 'cas')
	213	+ shutil.rmtree(cas)
214	214
215	215	result = cli.run(project=project, args=['artifact', 'pull', 'target.bst', '--remote',
216	216	good_share.repo])
...	...	@@ -250,8 +250,8 @@ def test_push_pull_non_strict(cli, tmpdir, datafiles):
250	250	# Now we've pushed, delete the user's local artifact cache
251	251	# directory and try to redownload it from the share
252	252	#
253		- artifacts = os.path.join(cli.directory, 'artifacts')
254		- shutil.rmtree(artifacts)
	253	+ cas = os.path.join(cli.directory, 'cas')
	254	+ shutil.rmtree(cas)
255	255
256	256	# Assert that nothing is cached locally anymore
257	257	for element_name in all_elements:
...	...	@@ -300,8 +300,8 @@ def test_push_pull_track_non_strict(cli, tmpdir, datafiles):
300	300	# Now we've pushed, delete the user's local artifact cache
301	301	# directory and try to redownload it from the share
302	302	#
303		- artifacts = os.path.join(cli.directory, 'artifacts')
304		- shutil.rmtree(artifacts)
	303	+ cas = os.path.join(cli.directory, 'cas')
	304	+ shutil.rmtree(cas)
305	305
306	306	# Assert that nothing is cached locally anymore
307	307	for element_name in all_elements:
...	...	@@ -336,7 +336,7 @@ def test_push_pull_cross_junction(cli, tmpdir, datafiles):
336	336	result.assert_success()
337	337	assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached'
338	338
339		- cache_dir = os.path.join(project, 'cache', 'artifacts')
	339	+ cache_dir = os.path.join(project, 'cache', 'cas')
340	340	shutil.rmtree(cache_dir)
341	341
342	342	assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable'
...	...	@@ -371,8 +371,8 @@ def test_pull_missing_blob(cli, tmpdir, datafiles):
371	371	# Now we've pushed, delete the user's local artifact cache
372	372	# directory and try to redownload it from the share
373	373	#
374		- artifacts = os.path.join(cli.directory, 'artifacts')
375		- shutil.rmtree(artifacts)
	374	+ cas = os.path.join(cli.directory, 'cas')
	375	+ shutil.rmtree(cas)
376	376
377	377	# Assert that nothing is cached locally anymore
378	378	for element_name in all_elements:

tests/integration/build-tree.py

---

...	...	@@ -124,10 +124,7 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
124	124	assert cli.get_element_state(project, element_name) == 'cached'
125	125
126	126	# Discard the cache
127		- cli.configure({
128		- 'artifacts': {'url': share.repo, 'push': True},
129		- 'artifactdir': os.path.join(cli.directory, 'artifacts2')
130		- })
	127	+ shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
131	128	assert cli.get_element_state(project, element_name) != 'cached'
132	129
133	130	# Pull from cache, ensuring cli options is set to pull the buildtree
...	...	@@ -160,10 +157,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
160	157	assert share.has_artifact('test', element_name, cli.get_element_key(project, element_name))
161	158
162	159	# Discard the cache
163		- cli.configure({
164		- 'artifacts': {'url': share.repo, 'push': True},
165		- 'artifactdir': os.path.join(cli.directory, 'artifacts2')
166		- })
	160	+ shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
167	161	assert cli.get_element_state(project, element_name) != 'cached'
168	162
169	163	# Pull from cache, but do not include buildtrees.
...	...	@@ -203,7 +197,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
203	197	])
204	198	assert 'Attempting to fetch missing artifact buildtree' in res.stderr
205	199	assert 'Hi' in res.output
206		- shutil.rmtree(os.path.join(os.path.join(cli.directory, 'artifacts2')))
	200	+ shutil.rmtree(os.path.join(os.path.join(str(tmpdir), 'cache', 'cas')))
207	201	assert cli.get_element_state(project, element_name) != 'cached'
208	202
209	203	# Check it's not loading the shell at all with always set for the buildtree, when the

tests/integration/cachedfail.py

---

...	...	@@ -160,7 +160,6 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
160	160
161	161	# This element should have failed
162	162	assert cli.get_element_state(project, 'element.bst') == 'failed'
163		- # This element should have been pushed to the remote
164	163	assert share.has_artifact('test', 'element.bst', cli.get_element_key(project, 'element.bst'))
165	164
166	165

tests/integration/messages.py

---

...	...	@@ -39,7 +39,7 @@ DATA_DIR = os.path.join(
39	39	@pytest.mark.integration
40	40	@pytest.mark.datafiles(DATA_DIR)
41	41	@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
42		-def test_disable_message_lines(cli, tmpdir, datafiles):
	42	+def test_disable_message_lines(cli, tmpdir, datafiles, integration_cache):
43	43	project = os.path.join(datafiles.dirname, datafiles.basename)
44	44	element_path = os.path.join(project, 'elements')
45	45	element_name = 'message.bst'
...	...	@@ -65,7 +65,7 @@ def test_disable_message_lines(cli, tmpdir, datafiles):
65	65	assert 'echo "Silly message"' in result.stderr
66	66
67	67	# Let's now build it again, but with --message-lines 0
68		- cli.remove_artifact_from_cache(project, element_name)
	68	+ cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
69	69	result = cli.run(project=project, args=["--message-lines", "0",
70	70	"build", element_name])
71	71	result.assert_success()
...	...	@@ -75,7 +75,7 @@ def test_disable_message_lines(cli, tmpdir, datafiles):
75	75	@pytest.mark.integration
76	76	@pytest.mark.datafiles(DATA_DIR)
77	77	@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
78		-def test_disable_error_lines(cli, tmpdir, datafiles):
	78	+def test_disable_error_lines(cli, tmpdir, datafiles, integration_cache):
79	79	project = os.path.join(datafiles.dirname, datafiles.basename)
80	80	element_path = os.path.join(project, 'elements')
81	81	element_name = 'message.bst'
...	...	@@ -102,7 +102,7 @@ def test_disable_error_lines(cli, tmpdir, datafiles):
102	102	assert "This is a syntax error" in result.stderr
103	103
104	104	# Let's now build it again, but with --error-lines 0
105		- cli.remove_artifact_from_cache(project, element_name)
	105	+ cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
106	106	result = cli.run(project=project, args=["--error-lines", "0",
107	107	"build", element_name])
108	108	result.assert_main_error(ErrorDomain.STREAM, None)

tests/integration/pullbuildtrees.py

---

...	...	@@ -21,10 +21,11 @@ DATA_DIR = os.path.join(
21	21	# to false, which is the default user context. The cache has to be
22	22	# cleared as just forcefully removing the refpath leaves dangling objects.
23	23	def default_state(cli, tmpdir, share):
24		- shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
	24	+ shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
	25	+ shutil.rmtree(os.path.join(str(tmpdir), 'extract'))
25	26	cli.configure({
26	27	'artifacts': {'url': share.repo, 'push': False},
27		- 'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
	28	+ 'cachedir': str(tmpdir),
28	29	'cache': {'pull-buildtrees': False},
29	30	})
30	31
...	...	@@ -45,7 +46,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
45	46	create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
46	47	cli2.configure({
47	48	'artifacts': {'url': share1.repo, 'push': True},
48		- 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
	49	+ 'cachedir': str(tmpdir),
49	50	})
50	51
51	52	# Build autotools element, checked pushed, delete local
...	...	@@ -74,8 +75,9 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
74	75	result = cli2.run(project=project, args=['artifact', 'pull', element_name])
75	76	assert element_name in result.get_pulled_elements()
76	77	elementdigest = share1.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
77		- buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
	78	+ buildtreedir = os.path.join(str(tmpdir), 'extract', 'test', 'autotools-amhello',
78	79	elementdigest.hash, 'buildtree')
	80	+ print("elementdigest: {}".format(elementdigest.hash))
79	81	assert not os.path.isdir(buildtreedir)
80	82	result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
81	83	assert element_name in result.get_pulled_elements()

tests/integration/source-determinism.py

---

...	...	@@ -94,9 +94,7 @@ def test_deterministic_source_umask(cli, tmpdir, datafiles, kind, integration_ca
94	94	return f.read()
95	95	finally:
96	96	os.umask(old_umask)
97		- cache_dir = integration_cache.artifacts
98		- cli.remove_artifact_from_cache(project, element_name,
99		- cache_dir=cache_dir)
	97	+ cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
100	98
101	99	assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
102	100
...	...	@@ -156,8 +154,6 @@ def test_deterministic_source_local(cli, tmpdir, datafiles, integration_cache):
156	154	with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
157	155	return f.read()
158	156	finally:
159		- cache_dir = integration_cache.artifacts
160		- cli.remove_artifact_from_cache(project, element_name,
161		- cache_dir=cache_dir)
	157	+ cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
162	158
163	159	assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)

tests/internals/context.py

---

...	...	@@ -43,7 +43,7 @@ def test_context_load(context_fixture):
43	43	context.load(config=os.devnull)
44	44	assert(context.sourcedir == os.path.join(cache_home, 'buildstream', 'sources'))
45	45	assert(context.builddir == os.path.join(cache_home, 'buildstream', 'build'))
46		- assert(context.artifactdir == os.path.join(cache_home, 'buildstream', 'artifacts'))
	46	+ assert(context.cachedir == os.path.join(cache_home, 'buildstream'))
47	47	assert(context.logdir == os.path.join(cache_home, 'buildstream', 'logs'))
48	48
49	49
...	...	@@ -57,7 +57,7 @@ def test_context_load_envvar(context_fixture):
57	57	context.load(config=os.devnull)
58	58	assert(context.sourcedir == os.path.join('/', 'some', 'path', 'buildstream', 'sources'))
59	59	assert(context.builddir == os.path.join('/', 'some', 'path', 'buildstream', 'build'))
60		- assert(context.artifactdir == os.path.join('/', 'some', 'path', 'buildstream', 'artifacts'))
	60	+ assert(context.cachedir == os.path.join('/', 'some', 'path', 'buildstream'))
61	61	assert(context.logdir == os.path.join('/', 'some', 'path', 'buildstream', 'logs'))
62	62
63	63	# Reset the environment variable
...	...	@@ -79,7 +79,7 @@ def test_context_load_user_config(context_fixture, datafiles):
79	79
80	80	assert(context.sourcedir == os.path.expanduser('~/pony'))
81	81	assert(context.builddir == os.path.join(cache_home, 'buildstream', 'build'))
82		- assert(context.artifactdir == os.path.join(cache_home, 'buildstream', 'artifacts'))
	82	+ assert(context.cachedir == os.path.join(cache_home, 'buildstream'))
83	83	assert(context.logdir == os.path.join(cache_home, 'buildstream', 'logs'))
84	84
85	85

tests/internals/pluginloading.py

---

...	...	@@ -16,7 +16,7 @@ def create_pipeline(tmpdir, basedir, target):
16	16	context = Context()
17	17	context.load(config=os.devnull)
18	18	context.deploydir = os.path.join(str(tmpdir), 'deploy')
19		- context.artifactdir = os.path.join(str(tmpdir), 'artifact')
	19	+ context.casdir = os.path.join(str(tmpdir), 'cas')
20	20	project = Project(basedir, context)
21	21
22	22	def dummy_handler(message, context):

tests/testutils/artifactshare.py

---

...	...	@@ -46,7 +46,6 @@ class ArtifactShare():
46	46	# in tests as a remote artifact push/pull configuration
47	47	#
48	48	self.repodir = os.path.join(self.directory, 'repo')
49		-
50	49	os.makedirs(self.repodir)
51	50
52	51	self.cas = CASCache(self.repodir)
...	...	@@ -171,7 +170,9 @@ class ArtifactShare():
171	170
172	171	def _mock_statvfs(self, path):
173	172	repo_size = 0
174		- for root, _, files in os.walk(self.repodir):
	173	+ for root, dirs, files in os.walk(self.repodir):
	174	+ for dirname in dirs:
	175	+ repo_size += os.path.getsize(os.path.join(root, dirname))
175	176	for filename in files:
176	177	repo_size += os.path.getsize(os.path.join(root, filename))
177	178

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.