Tristan Van Berkom pushed to branch bst-1.2 at BuildStream / buildstream
Commits:
-
3e1ea8d9
by Valentin David at 2018-08-23T08:04:27Z
-
7c84d089
by Tristan Van Berkom at 2018-08-23T09:03:18Z
2 changed files:
Changes:
| ... | ... | @@ -29,6 +29,8 @@ from urllib.parse import urlparse |
| 29 | 29 |
|
| 30 | 30 |
import grpc
|
| 31 | 31 |
|
| 32 |
+from .. import _yaml
|
|
| 33 |
+ |
|
| 32 | 34 |
from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
|
| 33 | 35 |
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
|
| 34 | 36 |
from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
|
| ... | ... | @@ -522,6 +524,25 @@ class CASCache(ArtifactCache): |
| 522 | 524 |
#
|
| 523 | 525 |
def remove(self, ref, *, defer_prune=False):
|
| 524 | 526 |
|
| 527 |
+ # Remove extract if not used by other ref
|
|
| 528 |
+ tree = self.resolve_ref(ref)
|
|
| 529 |
+ ref_name, ref_hash = os.path.split(ref)
|
|
| 530 |
+ extract = os.path.join(self.extractdir, ref_name, tree.hash)
|
|
| 531 |
+ keys_file = os.path.join(extract, 'meta', 'keys.yaml')
|
|
| 532 |
+ if os.path.exists(keys_file):
|
|
| 533 |
+ keys_meta = _yaml.load(keys_file)
|
|
| 534 |
+ keys = [keys_meta['strong'], keys_meta['weak']]
|
|
| 535 |
+ remove_extract = True
|
|
| 536 |
+ for other_hash in keys:
|
|
| 537 |
+ if other_hash == ref_hash:
|
|
| 538 |
+ continue
|
|
| 539 |
+ remove_extract = False
|
|
| 540 |
+ break
|
|
| 541 |
+ |
|
| 542 |
+ if remove_extract:
|
|
| 543 |
+ utils._force_rmtree(extract)
|
|
| 544 |
+ |
|
| 545 |
+ # Remove cache ref
|
|
| 525 | 546 |
refpath = self._refpath(ref)
|
| 526 | 547 |
if not os.path.exists(refpath):
|
| 527 | 548 |
raise ArtifactError("Could not find artifact for ref '{}'".format(ref))
|
| ... | ... | @@ -247,3 +247,38 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success): |
| 247 | 247 |
res.assert_success()
|
| 248 | 248 |
else:
|
| 249 | 249 |
res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|
| 250 |
+ |
|
| 251 |
+ |
|
| 252 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
| 253 |
+def test_extract_expiry(cli, datafiles, tmpdir):
|
|
| 254 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 255 |
+ element_path = 'elements'
|
|
| 256 |
+ |
|
| 257 |
+ cli.configure({
|
|
| 258 |
+ 'cache': {
|
|
| 259 |
+ 'quota': 10000000,
|
|
| 260 |
+ }
|
|
| 261 |
+ })
|
|
| 262 |
+ |
|
| 263 |
+ create_element_size('target.bst', project, element_path, [], 6000000)
|
|
| 264 |
+ res = cli.run(project=project, args=['build', 'target.bst'])
|
|
| 265 |
+ res.assert_success()
|
|
| 266 |
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
|
|
| 267 |
+ |
|
| 268 |
+ # Force creating extract
|
|
| 269 |
+ res = cli.run(project=project, args=['checkout', 'target.bst', os.path.join(str(tmpdir), 'checkout')])
|
|
| 270 |
+ res.assert_success()
|
|
| 271 |
+ |
|
| 272 |
+ extractdir = os.path.join(project, 'cache', 'artifacts', 'extract', 'test', 'target')
|
|
| 273 |
+ extracts = os.listdir(extractdir)
|
|
| 274 |
+ assert(len(extracts) == 1)
|
|
| 275 |
+ extract = os.path.join(extractdir, extracts[0])
|
|
| 276 |
+ |
|
| 277 |
+ # Remove target.bst from artifact cache
|
|
| 278 |
+ create_element_size('target2.bst', project, element_path, [], 6000000)
|
|
| 279 |
+ res = cli.run(project=project, args=['build', 'target2.bst'])
|
|
| 280 |
+ res.assert_success()
|
|
| 281 |
+ assert cli.get_element_state(project, 'target.bst') != 'cached'
|
|
| 282 |
+ |
|
| 283 |
+ # Now the extract should be removed.
|
|
| 284 |
+ assert not os.path.exists(extract)
|