[Notes] [Git][BuildStream/buildstream][tpollard/774] _stream.py: Ability to pull missing buildtrees outside of pull/build

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/774 at BuildStream / buildstream

Commits:

• 406f546f
  by Tom Pollard at 2018-11-30T11:46:58Z

  _stream.py: Ability to pull missing buildtrees outside of pull/build
  
  Adds helper function _buildtree_pull_required() to determine if a
  pullqueue should be constructed, for commands outside of bst pull
  and build where it is determined that an element's buildtree
  artifact is to be required given the respective semantics and
  config. Utilised in push() to attempt to mitigate skipping the push
  of partial elements without the user having to have preceded it with
  an explicit pull.
  
  cli.py: Add new behaviour to push command description
  
  element.py: Move _cached_buildtree() to be non local private method,
  use _KeyStrength types to reduce duplication.
  
  tests/integration/pullbuildtrees.py also updated to cover this
  use-case.
  

4 changed files:

• buildstream/_frontend/cli.py
• buildstream/_stream.py
• buildstream/element.py
• tests/integration/pullbuildtrees.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -469,6 +469,10 @@ def push(app, elements, deps, remote):
469	469	The default destination is the highest priority configured cache. You can
470	470	override this by passing a different cache URL with the `--remote` flag.
471	471
	472	+ If bst has been configured to include build trees on artifact pulls,
	473	+ an attempt will be made to pull any required build trees to avoid the
	474	+ skipping of partial artifacts being pushed.
	475	+
472	476	Specify `--deps` to control which artifacts to push:
473	477
474	478	\b

buildstream/_stream.py

---

...	...	@@ -327,6 +327,10 @@ class Stream():
327	327	# If `remote` specified as None, then regular configuration will be used
328	328	# to determine where to push artifacts to.
329	329	#
	330	+ # If any of the given targets are missing their expected buildtree artifact,
	331	+ # a pull queue will be created if user context and available remotes allow for
	332	+ # attempting to fetch them.
	333	+ #
330	334	def push(self, targets, *,
331	335	selection=PipelineSelection.NONE,
332	336	remote=None):
...	...	@@ -345,8 +349,17 @@ class Stream():
345	349	raise StreamError("No artifact caches available for pushing artifacts")
346	350
347	351	self._pipeline.assert_consistent(elements)
348		- self._add_queue(PushQueue(self._scheduler))
349		- self._enqueue_plan(elements)
	352	+
	353	+ # Check if we require a pull queue, with given artifact state and context
	354	+ require_buildtrees = self._buildtree_pull_required(elements)
	355	+ if require_buildtrees:
	356	+ self._message(MessageType.INFO, "Attempting to fetch missing artifact buildtrees")
	357	+ self._add_queue(PullQueue(self._scheduler))
	358	+ self._enqueue_plan(require_buildtrees)
	359	+
	360	+ push_queue = PushQueue(self._scheduler)
	361	+ self._add_queue(push_queue)
	362	+ self._enqueue_plan(elements, queue=push_queue)
350	363	self._run()
351	364
352	365	# checkout()
...	...	@@ -1237,3 +1250,28 @@ class Stream():
1237	1250	parts.append(element.normal_name)
1238	1251
1239	1252	return os.path.join(directory, *reversed(parts))
	1253	+
	1254	+ # _buildtree_pull_required()
	1255	+ #
	1256	+ # Check if current task, given config, requires element buildtree artifact
	1257	+ #
	1258	+ # Args:
	1259	+ # elements (list): elements to check if buildtrees are required
	1260	+ #
	1261	+ # Returns:
	1262	+ # (list): elements requiring buildtrees
	1263	+ #
	1264	+ def _buildtree_pull_required(self, elements):
	1265	+ required_list = []
	1266	+
	1267	+ # If context is set to not pull buildtrees, or no fetch remotes, return empty list
	1268	+ if not (self._context.pull_buildtrees or self._artifacts.has_fetch_remotes()):
	1269	+ return required_list
	1270	+
	1271	+ for element in elements:
	1272	+ # Check if element is partially cached without its buildtree, as the element
	1273	+ # artifact may not be cached at all
	1274	+ if element._cached() and not element._cached_buildtree():
	1275	+ required_list.append(element)
	1276	+
	1277	+ return required_list

buildstream/element.py

---

...	...	@@ -1422,7 +1422,7 @@ class Element(Plugin):
1422	1422	.format(workspace.get_absolute_path())):
1423	1423	workspace.stage(temp_staging_directory)
1424	1424	# Check if we have a cached buildtree to use
1425		- elif self.__cached_buildtree():
	1425	+ elif self._cached_buildtree():
1426	1426	artifact_base, _ = self.__extract()
1427	1427	import_dir = os.path.join(artifact_base, 'buildtree')
1428	1428	else:
...	...	@@ -1808,7 +1808,7 @@ class Element(Plugin):
1808	1808
1809	1809	# Do not push elements that aren't cached, or that are cached with a dangling buildtree
1810	1810	# artifact unless element type is expected to have an an empty buildtree directory
1811		- if not self.__cached_buildtree():
	1811	+ if not self._cached_buildtree():
1812	1812	return True
1813	1813
1814	1814	# Do not push tainted artifact
...	...	@@ -1998,6 +1998,29 @@ class Element(Plugin):
1998	1998	def _get_source_element(self):
1999	1999	return self
2000	2000
	2001	+ # _cached_buildtree()
	2002	+ #
	2003	+ # Check if element artifact contains expected buildtree. An
	2004	+ # element's buildtree artifact will not be present if the rest
	2005	+ # of the partial artifact is not cached.
	2006	+ #
	2007	+ # Returns:
	2008	+ # (bool): True if artifact cached with buildtree, False if
	2009	+ # element not cached or missing expected buildtree.
	2010	+ #
	2011	+ def _cached_buildtree(self):
	2012	+ context = self._get_context()
	2013	+
	2014	+ if not self._cached():
	2015	+ return False
	2016	+
	2017	+ key_strength = _KeyStrength.STRONG if context.get_strict() else _KeyStrength.WEAK
	2018	+ if not self.__artifacts.contains_subdir_artifact(self, self._get_cache_key(strength=key_strength),
	2019	+ 'buildtree'):
	2020	+ return False
	2021	+
	2022	+ return True
	2023	+
2001	2024	#############################################################
2002	2025	# Private Local Methods #
2003	2026	#############################################################
...	...	@@ -2764,27 +2787,6 @@ class Element(Plugin):
2764	2787
2765	2788	return True
2766	2789
2767		- # __cached_buildtree():
2768		- #
2769		- # Check if cached element artifact contains expected buildtree
2770		- #
2771		- # Returns:
2772		- # (bool): True if artifact cached with buildtree, False if
2773		- # element not cached or missing expected buildtree
2774		- #
2775		- def __cached_buildtree(self):
2776		- context = self._get_context()
2777		-
2778		- if not self._cached():
2779		- return False
2780		- elif context.get_strict():
2781		- if not self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, 'buildtree'):
2782		- return False
2783		- elif not self.__artifacts.contains_subdir_artifact(self, self.__weak_cache_key, 'buildtree'):
2784		- return False
2785		-
2786		- return True
2787		-
2788	2790	# __pull_directories():
2789	2791	#
2790	2792	# Which directories to include or exclude given the current

tests/integration/pullbuildtrees.py

---

...	...	@@ -38,7 +38,8 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
38	38
39	39	# Create artifact shares for pull & push testing
40	40	with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
41		- create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2:
	41	+ create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
	42	+ create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
42	43	cli.configure({
43	44	'artifacts': {'url': share1.repo, 'push': True},
44	45	'artifactdir': os.path.join(str(tmpdir), 'artifacts')
...	...	@@ -123,6 +124,32 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
123	124	assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
124	125	default_state(cli, tmpdir, share1)
125	126
	127	+ # Assert that bst push will automatically attempt to pull a missing buildtree
	128	+ # if pull-buildtrees is set, however as share3 is the only defined remote and is empty,
	129	+ # assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the
	130	+ # artifact cannot be pushed.
	131	+ result = cli.run(project=project, args=['pull', element_name])
	132	+ assert element_name in result.get_pulled_elements()
	133	+ cli.configure({'artifacts': {'url': share3.repo, 'push': True}})
	134	+ result = cli.run(project=project, args=['--pull-buildtrees', 'push', element_name])
	135	+ assert "Attempting to fetch missing artifact buildtrees" in result.stderr
	136	+ assert element_name not in result.get_pulled_elements()
	137	+ assert not os.path.isdir(buildtreedir)
	138	+ assert element_name not in result.get_pushed_elements()
	139	+ assert not share3.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	140	+
	141	+ # Assert that if we add an extra remote that has the buildtree artfact cached, bst push will
	142	+ # automatically attempt to pull it and will be successful, leading to the full artifact being pushed
	143	+ # to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote,
	144	+ # without exlipictly requiring a bst pull.
	145	+ cli.configure({'artifacts': [{'url': share1.repo, 'push': False}, {'url': share3.repo, 'push': True}]})
	146	+ result = cli.run(project=project, args=['--pull-buildtrees', 'push', element_name])
	147	+ assert "Attempting to fetch missing artifact buildtrees" in result.stderr
	148	+ assert element_name in result.get_pulled_elements()
	149	+ assert os.path.isdir(buildtreedir)
	150	+ assert element_name in result.get_pushed_elements()
	151	+ assert share3.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	152	+
126	153
127	154	# Ensure that only valid pull-buildtrees boolean options make it through the loading
128	155	# process.

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.