[Notes] [Git][BuildStream/buildstream][chandan/junction-dependency-format] 3 commits: loader: Allow dependencies to use ":" to refer to junctioned elements

[Chandan Singh <gitlab mg gitlab com>]

Title: GitLab

Chandan Singh pushed to branch chandan/junction-dependency-format at BuildStream / buildstream

Commits:

• ad3a85b8
  by Chandan Singh at 2019-02-18T18:25:57Z

  loader: Allow dependencies to use ":" to refer to junctioned elements
  
  This will allow cross-junction dependencies to be listed as strings on a
  single line.
  
  As part of this, some logic around initializing `Dependency()` objects
  have been moved out of `_extract_depends_from_node()` method into the
  constructor of `Dependency()` class, to keep all related code in one
  place.
  
  * _loader/types.py: While initializing `Dependency` objects, attempt to
                      split filenames, only if no `junction` was specified
                      explicitly. If a `junction` was specified, then
                      filenames with `:` in their names will result in an
                      error.
  
  * _loader/loadelement.py: Refactor logic to initialize `Dependency()`
                            objects to move it to the `Dependency()`
                            constructor.
  
  * tests/frontend/buildcheckout.py: Add tests to ensure the above.
  
  * _versions.py: Bump BST_FORMAT_VERSION.
  
  Fixes #809.
  

• ab6ab8fc
  by Chandan Singh at 2019-02-18T18:25:57Z

  NEWS: Cross-junction dependencies can now be specified inline
  

• 50027bbb
  by Chandan Singh at 2019-02-18T18:25:57Z

  Document how to refer to cross-junction dependencies inline
  
  In the previous commit, we added support to express cross-junction
  dependencies inline as simple strings. Document it along with the
  version in which the feature was added.
  

7 changed files:

• NEWS
• buildstream/_loader/loadelement.py
• buildstream/_loader/types.py
• buildstream/_versions.py
• buildstream/plugins/elements/junction.py
• doc/source/format_declaring.rst
• tests/frontend/buildcheckout.py

Changes:

NEWS

---

...	...	@@ -70,6 +70,9 @@ buildstream 1.3.1
70	70	to avoid having to specify the dependency type for every entry in
71	71	'depends'.
72	72
	73	+ o Elements may now specify cross-junction dependencies as simple strings
	74	+ using the format '{junction-name}:{element-name}'.
	75	+
73	76	o Source plugins may now request access access to previous during track and
74	77	fetch by setting `BST_REQUIRES_PREVIOUS_SOURCES_TRACK` and/or
75	78	`BST_REQUIRES_PREVIOUS_SOURCES_FETCH` attributes.

buildstream/_loader/loadelement.py

---

...	...	@@ -18,7 +18,6 @@
18	18	# Tristan Van Berkom <tristan vanberkom codethink co uk>
19	19
20	20	# System imports
21		-from collections.abc import Mapping
22	21	from itertools import count
23	22
24	23	from pyroaring import BitMap, FrozenBitMap # pylint: disable=no-name-in-module
...	...	@@ -174,38 +173,7 @@ def _extract_depends_from_node(node, *, key=None):
174	173
175	174	for index, dep in enumerate(depends):
176	175	dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[index])
177		-
178		- if isinstance(dep, str):
179		- dependency = Dependency(dep, provenance=dep_provenance, dep_type=default_dep_type)
180		-
181		- elif isinstance(dep, Mapping):
182		- if default_dep_type:
183		- _yaml.node_validate(dep, ['filename', 'junction'])
184		- dep_type = default_dep_type
185		- else:
186		- _yaml.node_validate(dep, ['filename', 'type', 'junction'])
187		-
188		- # Make type optional, for this we set it to None
189		- dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
190		- if dep_type is None or dep_type == Symbol.ALL:
191		- dep_type = None
192		- elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
193		- provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
194		- raise LoadError(LoadErrorReason.INVALID_DATA,
195		- "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
196		- .format(provenance, dep_type))
197		-
198		- filename = _yaml.node_get(dep, str, Symbol.FILENAME)
199		- junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None)
200		- dependency = Dependency(filename,
201		- dep_type=dep_type,
202		- junction=junction,
203		- provenance=dep_provenance)
204		-
205		- else:
206		- raise LoadError(LoadErrorReason.INVALID_DATA,
207		- "{}: Dependency is not specified as a string or a dictionary".format(dep_provenance))
208		-
	176	+ dependency = Dependency(dep, dep_provenance, default_dep_type=default_dep_type)
209	177	output_deps.append(dependency)
210	178
211	179	# Now delete the field, we dont want it anymore

buildstream/_loader/types.py

---

...	...	@@ -17,6 +17,11 @@
17	17	# Authors:
18	18	# Tristan Van Berkom <tristan vanberkom codethink co uk>
19	19
	20	+from collections.abc import Mapping
	21	+
	22	+from .._exceptions import LoadError, LoadErrorReason
	23	+from .. import _yaml
	24	+
20	25
21	26	# Symbol():
22	27	#
...	...	@@ -56,9 +61,56 @@ class Symbol():
56	61	# dependency was declared
57	62	#
58	63	class Dependency():
59		- def __init__(self, name,
60		- dep_type=None, junction=None, provenance=None):
61		- self.name = name
62		- self.dep_type = dep_type
63		- self.junction = junction
	64	+ def __init__(self, dep, provenance, default_dep_type=None):
64	65	self.provenance = provenance
	66	+
	67	+ if isinstance(dep, str):
	68	+ self.name = dep
	69	+ self.dep_type = default_dep_type
	70	+ self.junction = None
	71	+
	72	+ elif isinstance(dep, Mapping):
	73	+ if default_dep_type:
	74	+ _yaml.node_validate(dep, ['filename', 'junction'])
	75	+ dep_type = default_dep_type
	76	+ else:
	77	+ _yaml.node_validate(dep, ['filename', 'type', 'junction'])
	78	+
	79	+ # Make type optional, for this we set it to None
	80	+ dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
	81	+ if dep_type is None or dep_type == Symbol.ALL:
	82	+ dep_type = None
	83	+ elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
	84	+ provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
	85	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	86	+ "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
	87	+ .format(provenance, dep_type))
	88	+
	89	+ self.name = _yaml.node_get(dep, str, Symbol.FILENAME)
	90	+ self.dep_type = dep_type
	91	+ self.junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None)
	92	+
	93	+ else:
	94	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	95	+ "{}: Dependency is not specified as a string or a dictionary".format(provenance))
	96	+
	97	+ # `:` characters are not allowed in filename if a junction was
	98	+ # explicitly specified
	99	+ if self.junction and ':' in self.name:
	100	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	101	+ "{}: Dependency {} contains `:` in its name. "
	102	+ "`:` characters are not allowed in filename when "
	103	+ "junction attribute is specified.".format(self.provenance, self.name))
	104	+
	105	+ # Name of the element should never contain more than one `:` characters
	106	+ if self.name.count(':') > 1:
	107	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	108	+ "{}: Dependency {} contains multiple `:` in its name. "
	109	+ "Recursive lookups for cross-junction elements is not "
	110	+ "allowed.".format(self.provenance, self.name))
	111	+
	112	+ # Attempt to split name if no junction was specified explicitly
	113	+ if not self.junction:
	114	+ junction_path = self.name.rsplit(":", 1)
	115	+ self.name = junction_path[-1]
	116	+ self.junction = None if len(junction_path) == 1 else junction_path[-2]

buildstream/_versions.py

---

...	...	@@ -23,7 +23,7 @@
23	23	# This version is bumped whenever enhancements are made
24	24	# to the `project.conf` format or the core element format.
25	25	#
26		-BST_FORMAT_VERSION = 21
	26	+BST_FORMAT_VERSION = 22
27	27
28	28
29	29	# The base BuildStream artifact version

buildstream/plugins/elements/junction.py

---

...	...	@@ -109,6 +109,8 @@ Junctions can configure options of the linked project. Options are never
109	109	implicitly inherited across junctions, however, variables can be used to
110	110	explicitly assign the same value to a subproject option.
111	111
	112	+.. _core_junction_nested:
	113	+
112	114	Nested Junctions
113	115	----------------
114	116	Junctions can be nested. That is, subprojects are allowed to have junctions on

doc/source/format_declaring.rst

---

...	...	@@ -401,6 +401,41 @@ Attributes:
401	401	The ``junction`` attribute is available since :ref:`format version 1 <project_format_version>`
402	402
403	403
	404	+Cross-junction dependencies
	405	+~~~~~~~~~~~~~~~~~~~~~~~~~~~
	406	+As mentioned above, cross-junction dependencies can be specified using the
	407	+``junction`` attribute. They can also be expressed as simple strings as a
	408	+convenience shorthand. You can refer to cross-junction elements using the
	409	+syntax ``{junction-name}:{element-name}``.
	410	+
	411	+For example, the following is logically same as the example above:
	412	+
	413	+.. code:: yaml
	414	+
	415	+ build-depends:
	416	+ - baseproject.bst:foo.bst
	417	+
	418	+Similarly, you can also refer to cross-junction elements via the ``filename``
	419	+attribute, like so:
	420	+
	421	+.. code:: yaml
	422	+
	423	+ depends:
	424	+ - filename: baseproject.bst:foo.bst
	425	+ type: build
	426	+
	427	+.. note::
	428	+
	429	+ BuildStream does not allow recursice lookups for junction elements. If a
	430	+ filename contains more than one ``:`` (colon) character, an error will be
	431	+ raised. See :ref:`nested junctions <core_junction_nested>` for more details
	432	+ on nested junctions.
	433	+
	434	+.. note::
	435	+
	436	+ This shorthand is available since :ref:`format version 22 <project_format_version>`
	437	+
	438	+
404	439	.. _format_dependencies_types:
405	440
406	441	Dependency types

tests/frontend/buildcheckout.py

---

...	...	@@ -729,3 +729,139 @@ def test_build_checkout_cross_junction(datafiles, cli, tmpdir):
729	729
730	730	filename = os.path.join(checkout, 'etc', 'animal.conf')
731	731	assert os.path.exists(filename)
	732	+
	733	+
	734	+@pytest.mark.datafiles(DATA_DIR)
	735	+def test_build_junction_short_notation(cli, tmpdir, datafiles):
	736	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	737	+ subproject_path = os.path.join(project, 'files', 'sub-project')
	738	+ junction_path = os.path.join(project, 'elements', 'junction.bst')
	739	+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
	740	+ workspace = os.path.join(cli.directory, 'workspace')
	741	+ checkout = os.path.join(cli.directory, 'checkout')
	742	+
	743	+ # Create a repo to hold the subproject and generate a junction element for it
	744	+ ref = generate_junction(tmpdir, subproject_path, junction_path)
	745	+
	746	+ # Create a stack element to depend on a cross junction element, using
	747	+ # colon (:) as the separator
	748	+ element = {
	749	+ 'kind': 'stack',
	750	+ 'depends': ['junction.bst:import-etc.bst']
	751	+ }
	752	+ _yaml.dump(element, element_path)
	753	+
	754	+ # Now try to build it, this should automatically result in fetching
	755	+ # the junction itself at load time.
	756	+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
	757	+ result.assert_success()
	758	+
	759	+ # Assert that it's cached now
	760	+ assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
	761	+
	762	+ # Now check it out
	763	+ result = cli.run(project=project, args=[
	764	+ 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
	765	+ ])
	766	+ result.assert_success()
	767	+
	768	+ # Assert the content of /etc/animal.conf
	769	+ filename = os.path.join(checkout, 'etc', 'animal.conf')
	770	+ assert os.path.exists(filename)
	771	+ with open(filename, 'r') as f:
	772	+ contents = f.read()
	773	+ assert contents == 'animal=Pony\n'
	774	+
	775	+
	776	+@pytest.mark.datafiles(DATA_DIR)
	777	+def test_build_junction_short_notation_filename(cli, tmpdir, datafiles):
	778	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	779	+ subproject_path = os.path.join(project, 'files', 'sub-project')
	780	+ junction_path = os.path.join(project, 'elements', 'junction.bst')
	781	+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
	782	+ checkout = os.path.join(cli.directory, 'checkout')
	783	+
	784	+ # Create a repo to hold the subproject and generate a junction element for it
	785	+ ref = generate_junction(tmpdir, subproject_path, junction_path)
	786	+
	787	+ # Create a stack element to depend on a cross junction element, using
	788	+ # colon (:) as the separator
	789	+ element = {
	790	+ 'kind': 'stack',
	791	+ 'depends': [{'filename': 'junction.bst:import-etc.bst'}]
	792	+ }
	793	+ _yaml.dump(element, element_path)
	794	+
	795	+ # Now try to build it, this should automatically result in fetching
	796	+ # the junction itself at load time.
	797	+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
	798	+ result.assert_success()
	799	+
	800	+ # Assert that it's cached now
	801	+ assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
	802	+
	803	+ # Now check it out
	804	+ result = cli.run(project=project, args=[
	805	+ 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
	806	+ ])
	807	+ result.assert_success()
	808	+
	809	+ # Assert the content of /etc/animal.conf
	810	+ filename = os.path.join(checkout, 'etc', 'animal.conf')
	811	+ assert os.path.exists(filename)
	812	+ with open(filename, 'r') as f:
	813	+ contents = f.read()
	814	+ assert contents == 'animal=Pony\n'
	815	+
	816	+
	817	+@pytest.mark.datafiles(DATA_DIR)
	818	+def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles):
	819	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	820	+ subproject_path = os.path.join(project, 'files', 'sub-project')
	821	+ junction_path = os.path.join(project, 'elements', 'junction.bst')
	822	+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
	823	+ checkout = os.path.join(cli.directory, 'checkout')
	824	+
	825	+ # Create a repo to hold the subproject and generate a junction element for it
	826	+ ref = generate_junction(tmpdir, subproject_path, junction_path)
	827	+
	828	+ # Create a stack element to depend on a cross junction element, using
	829	+ # colon (:) as the separator
	830	+ element = {
	831	+ 'kind': 'stack',
	832	+ 'depends': [{
	833	+ 'filename': 'junction.bst:import-etc.bst',
	834	+ 'junction': 'junction.bst',
	835	+ }]
	836	+ }
	837	+ _yaml.dump(element, element_path)
	838	+
	839	+ # Now try to build it, this should fail as filenames should not contain
	840	+ # `:` when junction is explicity specified
	841	+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
	842	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
	843	+
	844	+
	845	+@pytest.mark.datafiles(DATA_DIR)
	846	+def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles):
	847	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	848	+ subproject_path = os.path.join(project, 'files', 'sub-project')
	849	+ junction_path = os.path.join(project, 'elements', 'junction.bst')
	850	+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
	851	+ checkout = os.path.join(cli.directory, 'checkout')
	852	+
	853	+ # Create a repo to hold the subproject and generate a junction element for it
	854	+ ref = generate_junction(tmpdir, subproject_path, junction_path)
	855	+
	856	+ # Create a stack element to depend on a cross junction element, using
	857	+ # colon (:) as the separator
	858	+ element = {
	859	+ 'kind': 'stack',
	860	+ 'depends': ['junction.bst:import-etc.bst:foo.bst']
	861	+ }
	862	+ _yaml.dump(element, element_path)
	863	+
	864	+ # Now try to build it, this should fail as recursive lookups for
	865	+ # cross-junction elements is not allowed.
	866	+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
	867	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.