[Notes] [Git][BuildStream/buildstream][raoul/870-root-cache-dir] CASCache: Move cache check methods here

[Raoul Hidalgo Charman <gitlab mg gitlab com>]

Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/870-root-cache-dir at BuildStream / buildstream

Commits:

• 04abdb87
  by Raoul Hidalgo Charman at 2019-02-04T13:07:08Z

  CASCache: Move cache check methods here
  
  Still fairly WIP.
  
  Part of #870
  

14 changed files:

• buildstream/_artifactcache.py
• buildstream/_cas/__init__.py
• buildstream/_cas/cascache.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_frontend/status.py
• buildstream/_frontend/widget.py
• buildstream/_scheduler/jobs/cachesizejob.py
• buildstream/_scheduler/jobs/cleanupjob.py
• buildstream/_scheduler/jobs/job.py
• buildstream/utils.py
• tests/artifactcache/cache_size.py
• tests/artifactcache/expiry.py
• tests/testutils/artifactshare.py

Changes:

buildstream/_artifactcache.py

---

...	...	@@ -46,39 +46,6 @@ class ArtifactCacheSpec(CASRemoteSpec):
46	46	pass
47	47
48	48
49		-# ArtifactCacheUsage
50		-#
51		-# A simple object to report the current artifact cache
52		-# usage details.
53		-#
54		-# Note that this uses the user configured cache quota
55		-# rather than the internal quota with protective headroom
56		-# removed, to provide a more sensible value to display to
57		-# the user.
58		-#
59		-# Args:
60		-# artifacts (ArtifactCache): The artifact cache to get the status of
61		-#
62		-class ArtifactCacheUsage():
63		-
64		- def __init__(self, artifacts):
65		- context = artifacts.context
66		- self.quota_config = context.config_cache_quota # Configured quota
67		- self.quota_size = artifacts._cache_quota_original # Resolved cache quota in bytes
68		- self.used_size = artifacts.get_cache_size() # Size used by artifacts in bytes
69		- self.used_percent = 0 # Percentage of the quota used
70		- if self.quota_size is not None:
71		- self.used_percent = int(self.used_size * 100 / self.quota_size)
72		-
73		- # Formattable into a human readable string
74		- #
75		- def __str__(self):
76		- return "{} / {} ({}%)" \
77		- .format(utils._pretty_size(self.used_size, dec_places=1),
78		- self.quota_config,
79		- self.used_percent)
80		-
81		-
82	49	# An ArtifactCache manages artifacts.
83	50	#
84	51	# Args:
...	...	@@ -95,11 +62,7 @@ class ArtifactCache():
95	62	self.project_remote_specs = {}
96	63
97	64	self._required_elements = set() # The elements required for this session
98		- self._cache_size = None # The current cache size, sometimes it's an estimate
99		- self._cache_quota = None # The cache quota
100		- self._cache_quota_original = None # The cache quota as specified by the user, in bytes
101		- self._cache_quota_headroom = None # The headroom in bytes before reaching the quota or full disk
102		- self._cache_lower_threshold = None # The target cache size for a cleanup
	65	+
103	66	self._remotes_setup = False # Check to prevent double-setup of remotes
104	67
105	68	# Per-project list of _CASRemote instances.
...	...	@@ -110,8 +73,6 @@ class ArtifactCache():
110	73
111	74	os.makedirs(self.extractdir, exist_ok=True)
112	75
113		- self._calculate_cache_quota()
114		-
115	76	# get_artifact_fullname()
116	77	#
117	78	# Generate a full name for an artifact, including the
...	...	@@ -266,7 +227,7 @@ class ArtifactCache():
266	227	space_saved = 0
267	228
268	229	# Start off with an announcement with as much info as possible
269		- volume_size, volume_avail = self._get_cache_volume_size()
	230	+ volume_size, volume_avail = self.cas._get_cache_volume_size()
270	231	self._message(MessageType.STATUS, "Starting cache cleanup",
271	232	detail=("Elements required by the current build plan: {}\n" +
272	233	"User specified quota: {} ({})\n" +
...	...	@@ -274,8 +235,8 @@ class ArtifactCache():
274	235	"Cache volume: {} total, {} available")
275	236	.format(len(self._required_elements),
276	237	context.config_cache_quota,
277		- utils._pretty_size(self._cache_quota_original, dec_places=2),
278		- utils._pretty_size(self.get_cache_size(), dec_places=2),
	238	+ utils._pretty_size(self.cas._cache_quota, dec_places=2),
	239	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2),
279	240	utils._pretty_size(volume_size, dec_places=2),
280	241	utils._pretty_size(volume_avail, dec_places=2)))
281	242
...	...	@@ -292,9 +253,10 @@ class ArtifactCache():
292	253	])
293	254
294	255	# Do a real computation of the cache size once, just in case
295		- self.compute_cache_size()
	256	+ usage = self.cas.compute_cache_size()
	257	+ self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
296	258
297		- while self.get_cache_size() >= self._cache_lower_threshold:
	259	+ while self.cas.get_cache_size() >= self.cas._cache_lower_threshold:
298	260	try:
299	261	to_remove = artifacts.pop(0)
300	262	except IndexError:
...	...	@@ -311,7 +273,7 @@ class ArtifactCache():
311	273	"Please increase the cache-quota in {} and/or make more disk space."
312	274	.format(removed_ref_count,
313	275	utils._pretty_size(space_saved, dec_places=2),
314		- utils._pretty_size(self.get_cache_size(), dec_places=2),
	276	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2),
315	277	len(self._required_elements),
316	278	(context.config_origin or default_conf)))
317	279
...	...	@@ -337,7 +299,7 @@ class ArtifactCache():
337	299	to_remove))
338	300
339	301	# Remove the size from the removed size
340		- self.set_cache_size(self._cache_size - size)
	302	+ self.cas.set_cache_size(self.cas._cache_size - size)
341	303
342	304	# User callback
343	305	#
...	...	@@ -353,29 +315,12 @@ class ArtifactCache():
353	315	"Cache usage is now: {}")
354	316	.format(removed_ref_count,
355	317	utils._pretty_size(space_saved, dec_places=2),
356		- utils._pretty_size(self.get_cache_size(), dec_places=2)))
357		-
358		- return self.get_cache_size()
359		-
360		- # compute_cache_size()
361		- #
362		- # Computes the real artifact cache size by calling
363		- # the abstract calculate_cache_size() method.
364		- #
365		- # Returns:
366		- # (int): The size of the artifact cache.
367		- #
368		- def compute_cache_size(self):
369		- old_cache_size = self._cache_size
370		- new_cache_size = self.cas.calculate_cache_size()
371		-
372		- if old_cache_size != new_cache_size:
373		- self._cache_size = new_cache_size
	318	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2)))
374	319
375		- usage = ArtifactCacheUsage(self)
376		- self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
	320	+ return self.cas.get_cache_size()
377	321
378		- return self._cache_size
	322	+ def full(self):
	323	+ return self.cas.full()
379	324
380	325	# add_artifact_size()
381	326	#
...	...	@@ -386,71 +331,10 @@ class ArtifactCache():
386	331	# artifact_size (int): The size to add.
387	332	#
388	333	def add_artifact_size(self, artifact_size):
389		- cache_size = self.get_cache_size()
	334	+ cache_size = self.cas.get_cache_size()
390	335	cache_size += artifact_size
391	336
392		- self.set_cache_size(cache_size)
393		-
394		- # get_cache_size()
395		- #
396		- # Fetches the cached size of the cache, this is sometimes
397		- # an estimate and periodically adjusted to the real size
398		- # when a cache size calculation job runs.
399		- #
400		- # When it is an estimate, the value is either correct, or
401		- # it is greater than the actual cache size.
402		- #
403		- # Returns:
404		- # (int) An approximation of the artifact cache size, in bytes.
405		- #
406		- def get_cache_size(self):
407		-
408		- # If we don't currently have an estimate, figure out the real cache size.
409		- if self._cache_size is None:
410		- stored_size = self._read_cache_size()
411		- if stored_size is not None:
412		- self._cache_size = stored_size
413		- else:
414		- self.compute_cache_size()
415		-
416		- return self._cache_size
417		-
418		- # set_cache_size()
419		- #
420		- # Forcefully set the overall cache size.
421		- #
422		- # This is used to update the size in the main process after
423		- # having calculated in a cleanup or a cache size calculation job.
424		- #
425		- # Args:
426		- # cache_size (int): The size to set.
427		- #
428		- def set_cache_size(self, cache_size):
429		-
430		- assert cache_size is not None
431		-
432		- self._cache_size = cache_size
433		- self._write_cache_size(self._cache_size)
434		-
435		- # full()
436		- #
437		- # Checks if the artifact cache is full, either
438		- # because the user configured quota has been exceeded
439		- # or because the underlying disk is almost full.
440		- #
441		- # Returns:
442		- # (bool): True if the artifact cache is full
443		- #
444		- def full(self):
445		-
446		- if self.get_cache_size() > self._cache_quota:
447		- return True
448		-
449		- _, volume_avail = self._get_cache_volume_size()
450		- if volume_avail < self._cache_quota_headroom:
451		- return True
452		-
453		- return False
	337	+ self.cas.set_cache_size(cache_size)
454	338
455	339	# preflight():
456	340	#
...	...	@@ -896,142 +780,6 @@ class ArtifactCache():
896	780	with self.context.timed_activity("Initializing remote caches", silent_nested=True):
897	781	self.initialize_remotes(on_failure=remote_failed)
898	782
899		- # _write_cache_size()
900		- #
901		- # Writes the given size of the artifact to the cache's size file
902		- #
903		- # Args:
904		- # size (int): The size of the artifact cache to record
905		- #
906		- def _write_cache_size(self, size):
907		- assert isinstance(size, int)
908		- size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
909		- with utils.save_file_atomic(size_file_path, "w") as f:
910		- f.write(str(size))
911		-
912		- # _read_cache_size()
913		- #
914		- # Reads and returns the size of the artifact cache that's stored in the
915		- # cache's size file
916		- #
917		- # Returns:
918		- # (int): The size of the artifact cache, as recorded in the file
919		- #
920		- def _read_cache_size(self):
921		- size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
922		-
923		- if not os.path.exists(size_file_path):
924		- return None
925		-
926		- with open(size_file_path, "r") as f:
927		- size = f.read()
928		-
929		- try:
930		- num_size = int(size)
931		- except ValueError as e:
932		- raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format(
933		- size, size_file_path)) from e
934		-
935		- return num_size
936		-
937		- # _calculate_cache_quota()
938		- #
939		- # Calculates and sets the cache quota and lower threshold based on the
940		- # quota set in Context.
941		- # It checks that the quota is both a valid _expression_, and that there is
942		- # enough disk space to satisfy that quota
943		- #
944		- def _calculate_cache_quota(self):
945		- # Headroom intended to give BuildStream a bit of leeway.
946		- # This acts as the minimum size of cache_quota and also
947		- # is taken from the user requested cache_quota.
948		- #
949		- if 'BST_TEST_SUITE' in os.environ:
950		- self._cache_quota_headroom = 0
951		- else:
952		- self._cache_quota_headroom = 2e9
953		-
954		- try:
955		- cache_quota = utils._parse_size(self.context.config_cache_quota,
956		- self.context.artifactdir)
957		- except utils.UtilError as e:
958		- raise LoadError(LoadErrorReason.INVALID_DATA,
959		- "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
960		- "\nValid values are, for example: 800M 10G 1T 50%\n"
961		- .format(str(e))) from e
962		-
963		- total_size, available_space = self._get_cache_volume_size()
964		- cache_size = self.get_cache_size()
965		-
966		- # Ensure system has enough storage for the cache_quota
967		- #
968		- # If cache_quota is none, set it to the maximum it could possibly be.
969		- #
970		- # Also check that cache_quota is at least as large as our headroom.
971		- #
972		- if cache_quota is None: # Infinity, set to max system storage
973		- cache_quota = cache_size + available_space
974		- if cache_quota < self._cache_quota_headroom: # Check minimum
975		- raise LoadError(LoadErrorReason.INVALID_DATA,
976		- "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
977		- "BuildStream requires a minimum cache quota of 2G.")
978		- elif cache_quota > total_size:
979		- # A quota greater than the total disk size is certianly an error
980		- raise ArtifactError("Your system does not have enough available " +
981		- "space to support the cache quota specified.",
982		- detail=("You have specified a quota of {quota} total disk space.\n" +
983		- "The filesystem containing {local_cache_path} only " +
984		- "has {total_size} total disk space.")
985		- .format(
986		- quota=self.context.config_cache_quota,
987		- local_cache_path=self.context.artifactdir,
988		- total_size=utils._pretty_size(total_size)),
989		- reason='insufficient-storage-for-quota')
990		- elif cache_quota > cache_size + available_space:
991		- # The quota does not fit in the available space, this is a warning
992		- if '%' in self.context.config_cache_quota:
993		- available = (available_space / total_size) * 100
994		- available = '{}% of total disk space'.format(round(available, 1))
995		- else:
996		- available = utils._pretty_size(available_space)
997		-
998		- self._message(MessageType.WARN,
999		- "Your system does not have enough available " +
1000		- "space to support the cache quota specified.",
1001		- detail=("You have specified a quota of {quota} total disk space.\n" +
1002		- "The filesystem containing {local_cache_path} only " +
1003		- "has {available_size} available.")
1004		- .format(quota=self.context.config_cache_quota,
1005		- local_cache_path=self.context.artifactdir,
1006		- available_size=available))
1007		-
1008		- # Place a slight headroom (2e9 (2GB) on the cache_quota) into
1009		- # cache_quota to try and avoid exceptions.
1010		- #
1011		- # Of course, we might still end up running out during a build
1012		- # if we end up writing more than 2G, but hey, this stuff is
1013		- # already really fuzzy.
1014		- #
1015		- self._cache_quota_original = cache_quota
1016		- self._cache_quota = cache_quota - self._cache_quota_headroom
1017		- self._cache_lower_threshold = self._cache_quota / 2
1018		-
1019		- # _get_cache_volume_size()
1020		- #
1021		- # Get the available space and total space for the volume on
1022		- # which the artifact cache is located.
1023		- #
1024		- # Returns:
1025		- # (int): The total number of bytes on the volume
1026		- # (int): The number of available bytes on the volume
1027		- #
1028		- # NOTE: We use this stub to allow the test cases
1029		- # to override what an artifact cache thinks
1030		- # about it's disk size and available bytes.
1031		- #
1032		- def _get_cache_volume_size(self):
1033		- return utils._get_volume_size(self.context.artifactdir)
1034		-
1035	783
1036	784	# _configured_remote_artifact_cache_specs():
1037	785	#

buildstream/_cas/__init__.py

---

...	...	@@ -17,5 +17,5 @@
17	17	# Authors:
18	18	# Tristan Van Berkom <tristan vanberkom codethink co uk>
19	19
20		-from .cascache import CASCache
	20	+from .cascache import CASCache, CASCacheUsage
21	21	from .casremote import CASRemote, CASRemoteSpec

buildstream/_cas/cascache.py

---

...	...	@@ -31,25 +31,69 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
31	31	from .._protos.buildstream.v2 import buildstream_pb2
32	32
33	33	from .. import utils
34		-from .._exceptions import CASCacheError
	34	+from .._exceptions import CASCacheError, LoadError, LoadErrorReason
35	35
36	36	from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate
37	37
38	38
	39	+CACHE_SIZE_FILE = "cache_size"
	40	+
	41	+
	42	+# CASCacheUsage
	43	+#
	44	+# A simple object to report the current CAS cache usage details.
	45	+#
	46	+# Note that this uses the user configured cache quota
	47	+# rather than the internal quota with protective headroom
	48	+# removed, to provide a more sensible value to display to
	49	+# the user.
	50	+#
	51	+# Args:
	52	+# cas (CASCache): The CAS cache to get the status of
	53	+#
	54	+class CASCacheUsage():
	55	+
	56	+ def __init__(self, cas):
	57	+ self.quota_config = cas._config_cache_quota # Configured quota
	58	+ self.quota_size = cas._cache_quota_original # Resolved cache quota in bytes
	59	+ self.used_size = cas.get_cache_size() # Size used by artifacts in bytes
	60	+ self.used_percent = 0 # Percentage of the quota used
	61	+ if self.quota_size is not None:
	62	+ self.used_percent = int(self.used_size * 100 / self.quota_size)
	63	+
	64	+ # Formattable into a human readable string
	65	+ #
	66	+ def __str__(self):
	67	+ return "{} / {} ({}%)" \
	68	+ .format(utils._pretty_size(self.used_size, dec_places=1),
	69	+ self.quota_config,
	70	+ self.used_percent)
	71	+
	72	+
39	73	# A CASCache manages a CAS repository as specified in the Remote Execution API.
40	74	#
41	75	# Args:
42	76	# path (str): The root directory for the CAS repository
	77	+# cache_quota (int): User configured cache quota
43	78	#
44	79	class CASCache():
45	80
46		- def __init__(self, path):
	81	+ def __init__(self, path, cache_quota=None):
47	82	self.casdir = os.path.join(path, 'cas')
48	83	self.tmpdir = os.path.join(path, 'tmp')
49	84	os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
50	85	os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
51	86	os.makedirs(self.tmpdir, exist_ok=True)
52	87
	88	+ self._config_cache_quota = cache_quota
	89	+ self._cache_size = None # The current cache size, sometimes it's an estimate
	90	+ self._cache_quota = None # The cache quota
	91	+ self._cache_quota_original = None # The cache quota as specified by the user, in bytes
	92	+ self._cache_quota_headroom = None
	93	+ self._cache_lower_threshold = None # The target cache size for a cleanup
	94	+
	95	+ self._calculate_cache_quota()
	96	+
53	97	# preflight():
54	98	#
55	99	# Preflight check.
...	...	@@ -587,6 +631,84 @@ class CASCache():
587	631	reachable = set()
588	632	self._reachable_refs_dir(reachable, tree, update_mtime=True)
589	633
	634	+ # compute_cache_size()
	635	+ #
	636	+ # Computes the real artifact cache size by calling
	637	+ # the abstract calculate_cache_size() method.
	638	+ #
	639	+ # Returns:
	640	+ # (int): The size of the artifact cache.
	641	+ #
	642	+ def compute_cache_size(self):
	643	+ old_cache_size = self._cache_size
	644	+ new_cache_size = self.calculate_cache_size()
	645	+
	646	+ if old_cache_size != new_cache_size:
	647	+ self._cache_size = new_cache_size
	648	+
	649	+ return self._cache_size
	650	+
	651	+ # get_cache_size()
	652	+ #
	653	+ # Fetches the cached size of the cache, this is sometimes
	654	+ # an estimate and periodically adjusted to the real size
	655	+ # when a cache size calculation job runs.
	656	+ #
	657	+ # When it is an estimate, the value is either correct, or
	658	+ # it is greater than the actual cache size.
	659	+ #
	660	+ # Returns:
	661	+ # (int) An approximation of the artifact cache size, in bytes.
	662	+ #
	663	+ def get_cache_size(self):
	664	+
	665	+ # If we don't currently have an estimate, figure out the real cache size.
	666	+ if self._cache_size is None:
	667	+ stored_size = self._read_cache_size()
	668	+ if stored_size is not None:
	669	+ self._cache_size = stored_size
	670	+ else:
	671	+ self._cache_size = self.compute_cache_size()
	672	+
	673	+ return self._cache_size
	674	+
	675	+ # set_cache_size()
	676	+ #
	677	+ # Forcefully set the overall cache size.
	678	+ #
	679	+ # This is used to update the size in the main process after
	680	+ # having calculated in a cleanup or a cache size calculation job.
	681	+ #
	682	+ # Args:
	683	+ # cache_size (int): The size to set.
	684	+ #
	685	+ def set_cache_size(self, cache_size):
	686	+
	687	+ assert cache_size is not None
	688	+
	689	+ self._cache_size = cache_size
	690	+ self._write_cache_size(self._cache_size)
	691	+
	692	+ # full()
	693	+ #
	694	+ # Checks if the artifact cache is full, either
	695	+ # because the user configured quota has been exceeded
	696	+ # or because the underlying disk is almost full.
	697	+ #
	698	+ # Returns:
	699	+ # (bool): True if the artifact cache is full
	700	+ #
	701	+ def full(self):
	702	+
	703	+ if self.get_cache_size() > self._cache_quota:
	704	+ return True
	705	+
	706	+ _, volume_avail = self._get_cache_volume_size()
	707	+ if volume_avail < self._cache_quota_headroom:
	708	+ return True
	709	+
	710	+ return False
	711	+
590	712	################################################
591	713	# Local Private Methods #
592	714	################################################
...	...	@@ -1015,6 +1137,131 @@ class CASCache():
1015	1137	# Send final batch
1016	1138	batch.send()
1017	1139
	1140	+ # _read_cache_size()
	1141	+ #
	1142	+ # Reads and returns the size of the artifact cache that's stored in the
	1143	+ # cache's size file
	1144	+ #
	1145	+ # Returns:
	1146	+ # (int): The size of the artifact cache, as recorded in the file
	1147	+ #
	1148	+ def _read_cache_size(self):
	1149	+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
	1150	+
	1151	+ if not os.path.exists(size_file_path):
	1152	+ return None
	1153	+
	1154	+ with open(size_file_path, "r") as f:
	1155	+ size = f.read()
	1156	+
	1157	+ try:
	1158	+ num_size = int(size)
	1159	+ except ValueError as e:
	1160	+ raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
	1161	+ size, size_file_path)) from e
	1162	+
	1163	+ return num_size
	1164	+
	1165	+ # _write_cache_size()
	1166	+ #
	1167	+ # Writes the given size of the artifact to the cache's size file
	1168	+ #
	1169	+ # Args:
	1170	+ # size (int): The size of the artifact cache to record
	1171	+ #
	1172	+ def _write_cache_size(self, size):
	1173	+ assert isinstance(size, int)
	1174	+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
	1175	+ with utils.save_file_atomic(size_file_path, "w") as f:
	1176	+ f.write(str(size))
	1177	+
	1178	+ # _get_cache_volume_size()
	1179	+ #
	1180	+ # Get the available space and total space for the volume on
	1181	+ # which the artifact cache is located.
	1182	+ #
	1183	+ # Returns:
	1184	+ # (int): The total number of bytes on the volume
	1185	+ # (int): The number of available bytes on the volume
	1186	+ #
	1187	+ # NOTE: We use this stub to allow the test cases
	1188	+ # to override what an artifact cache thinks
	1189	+ # about it's disk size and available bytes.
	1190	+ #
	1191	+ def _get_cache_volume_size(self):
	1192	+ return utils._get_volume_size(self.casdir)
	1193	+
	1194	+ # _calculate_cache_quota()
	1195	+ #
	1196	+ # Calculates and sets the cache quota and lower threshold based on the
	1197	+ # quota set in Context.
	1198	+ # It checks that the quota is both a valid _expression_, and that there is
	1199	+ # enough disk space to satisfy that quota
	1200	+ #
	1201	+ def _calculate_cache_quota(self):
	1202	+ # Headroom intended to give BuildStream a bit of leeway.
	1203	+ # This acts as the minimum size of cache_quota and also
	1204	+ # is taken from the user requested cache_quota.
	1205	+ #
	1206	+ if 'BST_TEST_SUITE' in os.environ:
	1207	+ self._cache_quota_headroom = 0
	1208	+ else:
	1209	+ self._cache_quota_headroom = 2e9
	1210	+
	1211	+
	1212	+ total_size, available_space = self._get_cache_volume_size()
	1213	+ cache_size = self.get_cache_size()
	1214	+
	1215	+ # Ensure system has enough storage for the cache_quota
	1216	+ #
	1217	+ # If cache_quota is none, set it to the maximum it could possibly be.
	1218	+ #
	1219	+ # Also check that cache_quota is at least as large as our headroom.
	1220	+ #
	1221	+ cache_quota = self._config_cache_quota
	1222	+ if cache_quota is None: # Infinity, set to max system storage
	1223	+ cache_quota = cache_size + available_space
	1224	+ if cache_quota < self._cache_quota_headroom: # Check minimum
	1225	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	1226	+ "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
	1227	+ "BuildStream requires a minimum cache quota of 2G.")
	1228	+ elif cache_quota > total_size:
	1229	+ # A quota greater than the total disk size is certianly an error
	1230	+ raise CASCacheError("Your system does not have enough available " +
	1231	+ "space to support the cache quota specified.",
	1232	+ detail=("You have specified a quota of {quota} total disk space.\n" +
	1233	+ "The filesystem containing {local_cache_path} only " +
	1234	+ "has {total_size} total disk space.")
	1235	+ .format(
	1236	+ quota=self._config_cache_quota,
	1237	+ local_cache_path=self.casdir,
	1238	+ total_size=utils._pretty_size(total_size)),
	1239	+ reason='insufficient-storage-for-quota')
	1240	+ elif cache_quota > cache_size + available_space:
	1241	+ # The quota does not fit in the available space, this is a warning
	1242	+ available = utils._pretty_size(available_space)
	1243	+
	1244	+ # self._message(MessageType.WARN,
	1245	+ # "Your system does not have enough available " +
	1246	+ # "space to support the cache quota specified.",
	1247	+ # detail=("You have specified a quota of {quota} total disk space.\n" +
	1248	+ # "The filesystem containing {local_cache_path} only " +
	1249	+ # "has {available_size} available.")
	1250	+ # .format(quota=self.config_cache_quota,
	1251	+ # local_cache_path=self.artifactdir,
	1252	+ # available_size=available))
	1253	+
	1254	+ # Place a slight headroom (2e9 (2GB) on the cache_quota) into
	1255	+ # cache_quota to try and avoid exceptions.
	1256	+ #
	1257	+ # Of course, we might still end up running out during a build
	1258	+ # if we end up writing more than 2G, but hey, this stuff is
	1259	+ # already really fuzzy.
	1260	+ #
	1261	+ self._cache_quota_original = cache_quota
	1262	+ self._cache_quota = cache_quota - self._cache_quota_headroom
	1263	+ self._cache_lower_threshold = self._cache_quota / 2
	1264	+
1018	1265
1019	1266	def _grouper(iterable, n):
1020	1267	while True:

buildstream/_context.py

---

...	...	@@ -30,8 +30,8 @@ from . import _yaml
30	30	from ._exceptions import LoadError, LoadErrorReason, BstError
31	31	from ._message import Message, MessageType
32	32	from ._profile import Topics, profile_start, profile_end
33		-from ._artifactcache import ArtifactCache, ArtifactCacheUsage
34		-from ._cas import CASCache
	33	+from ._artifactcache import ArtifactCache
	34	+from ._cas import CASCache, CASCacheUsage
35	35	from ._workspaces import Workspaces, WorkspaceProjectCache
36	36	from .plugin import _plugin_lookup
37	37	from .sandbox import SandboxRemote
...	...	@@ -237,7 +237,17 @@ class Context():
237	237	cache = _yaml.node_get(defaults, Mapping, 'cache')
238	238	_yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
239	239
240		- self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
	240	+ config_cache_quota = _yaml.node_get(cache, str, 'quota')
	241	+ os.makedirs(self.casdir, exist_ok=True)
	242	+ try:
	243	+ self.config_cache_quota = utils._parse_size(config_cache_quota,
	244	+ self.casdir)
	245	+ except utils.UtilError as e:
	246	+ print(config_cache_quota)
	247	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	248	+ "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
	249	+ "\nValid values are, for example: 800M 10G 1T 50%\n"
	250	+ .format(str(e))) from e
241	251
242	252	# Load artifact share configuration
243	253	self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
...	...	@@ -315,15 +325,15 @@ class Context():
315	325
316	326	return self._artifactcache
317	327
318		- # get_artifact_cache_usage()
	328	+ # get_cache_usage()
319	329	#
320	330	# Fetches the current usage of the artifact cache
321	331	#
322	332	# Returns:
323		- # (ArtifactCacheUsage): The current status
	333	+ # (CASCacheUsage): The current status
324	334	#
325		- def get_artifact_cache_usage(self):
326		- return ArtifactCacheUsage(self.artifactcache)
	335	+ def get_cache_usage(self):
	336	+ return CASCacheUsage(self.get_cascache())
327	337
328	338	# add_project():
329	339	#
...	...	@@ -690,7 +700,7 @@ class Context():
690	700
691	701	def get_cascache(self):
692	702	if self._cascache is None:
693		- self._cascache = CASCache(self.rootcachedir)
	703	+ self._cascache = CASCache(self.rootcachedir, self.config_cache_quota)
694	704	return self._cascache
695	705
696	706

buildstream/_frontend/cli.py

---

...	...	@@ -11,11 +11,11 @@ from .._exceptions import BstError, LoadError, AppError
11	11	from .._versions import BST_FORMAT_VERSION
12	12	from .complete import main_bashcomplete, complete_path, CompleteUnhandled
13	13
14		-
15	14	##################################################################
16	15	# Override of click's main entry point #
17	16	##################################################################
18	17
	18	+
19	19	# search_command()
20	20	#
21	21	# Helper function to get a command and context object

buildstream/_frontend/status.py

---

...	...	@@ -404,7 +404,7 @@ class _StatusHeader():
404	404	#
405	405	# ~~~~~~ cache: 69% ~~~~~~
406	406	#
407		- usage = self._context.get_artifact_cache_usage()
	407	+ usage = self._context.get_cache_usage()
408	408	usage_percent = '{}%'.format(usage.used_percent)
409	409
410	410	size = 21

buildstream/_frontend/widget.py

---

...	...	@@ -452,7 +452,7 @@ class LogLine(Widget):
452	452	values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
453	453	values["Project"] = "{} ({})".format(project.name, project.directory)
454	454	values["Targets"] = ", ".join([t.name for t in stream.targets])
455		- values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage())
	455	+ values["Cache Usage"] = "{}".format(context.get_cache_usage())
456	456	text += self._format_values(values)
457	457
458	458	# User configurations

buildstream/_scheduler/jobs/cachesizejob.py

---

...	...	@@ -25,14 +25,14 @@ class CacheSizeJob(Job):
25	25	self._complete_cb = complete_cb
26	26
27	27	context = self._scheduler.context
28		- self._artifacts = context.artifactcache
	28	+ self._cas = context.get_cascache()
29	29
30	30	def child_process(self):
31		- return self._artifacts.compute_cache_size()
	31	+ return self._cas.compute_cache_size()
32	32
33	33	def parent_complete(self, status, result):
34	34	if status == JobStatus.OK:
35		- self._artifacts.set_cache_size(result)
	35	+ self._cas.set_cache_size(result)
36	36
37	37	if self._complete_cb:
38	38	self._complete_cb(status, result)

buildstream/_scheduler/jobs/cleanupjob.py

---

...	...	@@ -25,27 +25,27 @@ class CleanupJob(Job):
25	25	self._complete_cb = complete_cb
26	26
27	27	context = self._scheduler.context
	28	+ self._cas = context.get_cascache()
28	29	self._artifacts = context.artifactcache
29	30
30	31	def child_process(self):
31	32	def progress():
32	33	self.send_message('update-cache-size',
33		- self._artifacts.get_cache_size())
	34	+ self._cas.get_cache_size())
34	35	return self._artifacts.clean(progress)
35	36
36	37	def handle_message(self, message_type, message):
37		-
38	38	# Update the cache size in the main process as we go,
39	39	# this provides better feedback in the UI.
40	40	if message_type == 'update-cache-size':
41		- self._artifacts.set_cache_size(message)
	41	+ self._cas.set_cache_size(message)
42	42	return True
43	43
44	44	return False
45	45
46	46	def parent_complete(self, status, result):
47	47	if status == JobStatus.OK:
48		- self._artifacts.set_cache_size(result)
	48	+ self._cas.set_cache_size(result)
49	49
50	50	if self._complete_cb:
51	51	self._complete_cb(status, result)

buildstream/_scheduler/jobs/job.py

---

...	...	@@ -381,7 +381,6 @@ class Job():
381	381	# queue (multiprocessing.Queue): The message queue for IPC
382	382	#
383	383	def _child_action(self, queue):
384		-
385	384	# This avoids some SIGTSTP signals from grandchildren
386	385	# getting propagated up to the master process
387	386	os.setsid()

buildstream/utils.py

---

...	...	@@ -43,6 +43,7 @@ from . import _signals
43	43	from ._exceptions import BstError, ErrorDomain
44	44	from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
45	45
	46	+
46	47	# The magic number for timestamps: 2011-11-11 11:11:11
47	48	_magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11])
48	49

tests/artifactcache/cache_size.py

---

...	...	@@ -50,14 +50,15 @@ def test_cache_size_write(cli, tmpdir):
50	50
51	51	# Artifact cache must be in a known place
52	52	artifactdir = os.path.join(project_dir, "artifacts")
53		- cli.configure({"artifactdir": artifactdir})
	53	+ casdir = os.path.join(project_dir, "cas")
	54	+ cli.configure({"rootcachedir": project_dir})
54	55
55	56	# Build, to populate the cache
56	57	res = cli.run(project=project_dir, args=["build", "test.bst"])
57	58	res.assert_success()
58	59
59	60	# Inspect the artifact cache
60		- sizefile = os.path.join(artifactdir, CACHE_SIZE_FILE)
	61	+ sizefile = os.path.join(casdir, CACHE_SIZE_FILE)
61	62	assert os.path.isfile(sizefile)
62	63	with open(sizefile, "r") as f:
63	64	size_data = f.read()
...	...	@@ -80,11 +81,11 @@ def test_quota_over_1024T(cli, tmpdir):
80	81	_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
81	82
82	83	volume_space_patch = mock.patch(
83		- "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
	84	+ "buildstream._cas.CASCache._get_cache_volume_size",
84	85	autospec=True,
85	86	return_value=(1025 * TiB, 1025 * TiB)
86	87	)
87	88
88	89	with volume_space_patch:
89	90	result = cli.run(project, args=["build", "file.bst"])
90		- result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
	91	+ result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota')

tests/artifactcache/expiry.py

---

...	...	@@ -340,11 +340,11 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
340	340	("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
341	341
342	342	# Not enough space on disk even if you cleaned up
343		- ("11K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'),
	343	+ ("11K", ErrorDomain.CAS, 'insufficient-storage-for-quota'),
344	344
345	345	# Not enough space for these caches
346		- ("7K", 'warning', 'Your system does not have enough available'),
347		- ("70%", 'warning', 'Your system does not have enough available')
	346	+ ("7K", 'success', None),
	347	+ ("70%", 'success', None)
348	348	])
349	349	@pytest.mark.datafiles(DATA_DIR)
350	350	def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reason):
...	...	@@ -372,13 +372,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
372	372	total_space = 10000
373	373
374	374	volume_space_patch = mock.patch(
375		- "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
	375	+ "buildstream._cas.CASCache._get_cache_volume_size",
376	376	autospec=True,
377	377	return_value=(total_space, free_space),
378	378	)
379	379
380	380	cache_size_patch = mock.patch(
381		- "buildstream._artifactcache.ArtifactCache.get_cache_size",
	381	+ "buildstream._cas.CASCache.get_cache_size",
382	382	autospec=True,
383	383	return_value=0,
384	384	)

tests/testutils/artifactshare.py

---

...	...	@@ -46,7 +46,6 @@ class ArtifactShare():
46	46	# in tests as a remote artifact push/pull configuration
47	47	#
48	48	self.repodir = os.path.join(self.directory, 'repo')
49		-
50	49	os.makedirs(self.repodir)
51	50
52	51	self.cas = CASCache(self.repodir)
...	...	@@ -171,7 +170,9 @@ class ArtifactShare():
171	170
172	171	def _mock_statvfs(self, path):
173	172	repo_size = 0
174		- for root, _, files in os.walk(self.repodir):
	173	+ for root, dirs, files in os.walk(self.repodir):
	174	+ for dirname in dirs:
	175	+ repo_size += os.path.getsize(os.path.join(root, dirname))
175	176	for filename in files:
176	177	repo_size += os.path.getsize(os.path.join(root, filename))
177	178

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.