[Notes] [Git][BuildStream/buildstream][jonathan/workspace-fragment-create] 12 commits: element.py: Pass bare_directory also to SandboxRemote()

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch jonathan/workspace-fragment-create at BuildStream / buildstream

Commits:

• 3788e701
  by Jürg Billeter at 2018-11-03T11:52:00Z

  element.py: Pass bare_directory also to SandboxRemote()
  
  Fixes: 825963b5 ("element: Make "--sysroot" take a bare directory")
  

• 82e971ef
  by Jürg Billeter at 2018-11-05T11:33:20Z

  Merge branch 'juerg/remote-execution' into 'master'
  
  element.py: Pass bare_directory also to SandboxRemote()
  
  See merge request BuildStream/buildstream!928

• 553696fe
  by Jonathan Maw at 2018-11-05T11:39:39Z

  _workspaces.py: Add a WorkspaceLocal object
  
  WorkspaceLocal contains all the information needed to generate data
  that's stored within the workspace. Currently this is just a list of
  project directories.
  

• 2787c5ed
  by Jonathan Maw at 2018-11-05T11:39:39Z

  _stream.py: Opened workspaces store the project that opened them
  

• 3aff2474
  by Jonathan Maw at 2018-11-05T11:39:39Z

  _stream.py: workspace_close deletes the stored project reference
  

• 3b6aa5c4
  by Jonathan Maw at 2018-11-05T11:39:40Z

  tests: Workspace tests check existence of .bstproject.yaml
  

• c0584183
  by Jonathan Maw at 2018-11-05T11:39:40Z

  _project.py: Rename _ensure_project_dir to _find_project_dir
  

• a030094e
  by Jonathan Maw at 2018-11-05T11:39:40Z

  utils.py: Add a helper for searching upwards for files
  
  i.e. with a given directory and filename, check parent directories until
  either a directory with the filename is found, or you reach the root of
  the filesystem.
  

• f6efd641
  by Jonathan Maw at 2018-11-05T11:39:40Z

  cli.py: Use utils' search upwards helper when searching for project.conf
  

• 128402d5
  by Jonathan Maw at 2018-11-05T11:39:40Z

  _project.py: Use utils' search upwards helper when searching for project.conf
  

• 392c14e6
  by Jonathan Maw at 2018-11-05T11:39:40Z

  tests: Try building from a workspace
  

• 9343a887
  by Jonathan Maw at 2018-11-05T11:39:40Z

  _project.py: Find project from workspace if outside of a project
  

7 changed files:

• buildstream/_frontend/cli.py
• buildstream/_project.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/element.py
• buildstream/utils.py
• tests/frontend/workspace.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -59,18 +59,9 @@ def complete_target(args, incomplete):
59	59	:return: all the possible user-specified completions for the param
60	60	"""
61	61
	62	+ from .. import utils
62	63	project_conf = 'project.conf'
63	64
64		- def ensure_project_dir(directory):
65		- directory = os.path.abspath(directory)
66		- while not os.path.isfile(os.path.join(directory, project_conf)):
67		- parent_dir = os.path.dirname(directory)
68		- if directory == parent_dir:
69		- break
70		- directory = parent_dir
71		-
72		- return directory
73		-
74	65	# First resolve the directory, in case there is an
75	66	# active --directory/-C option
76	67	#
...	...	@@ -89,7 +80,7 @@ def complete_target(args, incomplete):
89	80	else:
90	81	# Check if this directory or any of its parent directories
91	82	# contain a project config file
92		- base_directory = ensure_project_dir(base_directory)
	83	+ base_directory = utils._search_upward_for_file(base_directory, project_conf)
93	84
94	85	# Now parse the project.conf just to find the element path,
95	86	# this is unfortunately a bit heavy.

buildstream/_project.py

---

...	...	@@ -40,6 +40,7 @@ from .element import Element
40	40	from ._message import Message, MessageType
41	41	from ._includes import Includes
42	42	from ._platform import Platform
	43	+from ._workspaces import WorkspaceLocal
43	44
44	45
45	46	# Project Configuration file
...	...	@@ -95,7 +96,7 @@ class Project():
95	96	self.name = None
96	97
97	98	# The project directory
98		- self.directory = self._ensure_project_dir(directory)
	99	+ self.directory = self._find_project_dir(directory)
99	100
100	101	# Absolute path to where elements are loaded from within the project
101	102	self.element_path = None
...	...	@@ -647,7 +648,7 @@ class Project():
647	648	# Source url aliases
648	649	output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
649	650
650		- # _ensure_project_dir()
	651	+ # _find_project_dir()
651	652	#
652	653	# Returns path of the project directory, if a configuration file is found
653	654	# in given directory or any of its parent directories.
...	...	@@ -658,18 +659,19 @@ class Project():
658	659	# Raises:
659	660	# LoadError if project.conf is not found
660	661	#
661		- def _ensure_project_dir(self, directory):
662		- directory = os.path.abspath(directory)
663		- while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
664		- parent_dir = os.path.dirname(directory)
665		- if directory == parent_dir:
	662	+ def _find_project_dir(self, directory):
	663	+ project_directory = utils._search_upward_for_file(directory, _PROJECT_CONF_FILE)
	664	+ if not project_directory:
	665	+ workspace_local = WorkspaceLocal.load(directory)
	666	+ if workspace_local:
	667	+ project_directory = workspace_local.get_default_path()
	668	+ else:
666	669	raise LoadError(
667	670	LoadErrorReason.MISSING_PROJECT_CONF,
668	671	'{} not found in current directory or any of its parent directories'
669	672	.format(_PROJECT_CONF_FILE))
670		- directory = parent_dir
671	673
672		- return directory
	674	+ return project_directory
673	675
674	676	def _load_plugin_factories(self, config, output):
675	677	plugin_source_origins = [] # Origins of custom sources

buildstream/_stream.py

---

...	...	@@ -32,6 +32,7 @@ from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
32	32	from ._message import Message, MessageType
33	33	from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
34	34	from ._pipeline import Pipeline, PipelineSelection
	35	+from ._workspaces import WorkspaceLocal
35	36	from . import utils, _yaml, _site
36	37	from . import Scope, Consistency
37	38
...	...	@@ -516,6 +517,10 @@ class Stream():
516	517	with target.timed_activity("Staging sources to {}".format(directory)):
517	518	target._open_workspace()
518	519
	520	+ project = self._context.get_toplevel_project()
	521	+ workspace_local = WorkspaceLocal.create(directory, project.directory, target._get_full_name())
	522	+ workspace_local.write()
	523	+
519	524	workspaces.save_config()
520	525	self._message(MessageType.INFO, "Saved workspace configuration")
521	526
...	...	@@ -540,6 +545,11 @@ class Stream():
540	545	except OSError as e:
541	546	raise StreamError("Could not remove '{}': {}"
542	547	.format(workspace.get_absolute_path(), e)) from e
	548	+ else:
	549	+ # TODO: At some point, closing a workspace only deletes the file if no projects are using it.
	550	+ workspace_local = WorkspaceLocal.load(workspace.get_absolute_path())
	551	+ if workspace_local:
	552	+ workspace_local.delete()
543	553
544	554	# Delete the workspace and save the configuration
545	555	workspaces.delete_workspace(element_name)

buildstream/_workspaces.py

---

...	...	@@ -25,6 +25,140 @@ from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27	27	BST_WORKSPACE_FORMAT_VERSION = 3
	28	+BST_WORKSPACE_LOCAL_FORMAT_VERSION = 1
	29	+WORKSPACE_LOCAL_FILE = ".bstproject.yaml"
	30	+
	31	+
	32	+# WorkspaceLocal()
	33	+#
	34	+# An object to contain various helper functions and data required for
	35	+# referring from a workspace back to buildstream.
	36	+#
	37	+# Args:
	38	+# directory (str): The directory that the workspace exists in
	39	+# project_path (str): The project path used to refer back
	40	+# to buildstream projects.
	41	+# element_name (str): The name of the element used to create this workspace.
	42	+class WorkspaceLocal():
	43	+ def __init__(self, directory, project_path="", element_name=""):
	44	+ self._projects = []
	45	+ self._directory = directory
	46	+
	47	+ assert (project_path and element_name) or (not project_path and not element_name)
	48	+ if project_path:
	49	+ self._add_project(project_path, element_name)
	50	+
	51	+ # get_default_path()
	52	+ #
	53	+ # Retrieves the default path to a project.
	54	+ #
	55	+ # Returns:
	56	+ # (str): The path to a project
	57	+ def get_default_path(self):
	58	+ return self._projects[0]['project-path']
	59	+
	60	+ # to_dict()
	61	+ #
	62	+ # Turn the members data into a dict for serialization purposes
	63	+ #
	64	+ # Returns:
	65	+ # (dict): A dict representation of the WorkspaceLocal
	66	+ #
	67	+ def to_dict(self):
	68	+ ret = {
	69	+ 'projects': self._projects,
	70	+ 'format-version': BST_WORKSPACE_LOCAL_FORMAT_VERSION,
	71	+ }
	72	+ return ret
	73	+
	74	+ # from_dict()
	75	+ #
	76	+ # Loads a new WorkspaceLocal from a simple dictionary
	77	+ #
	78	+ # Args:
	79	+ # directory (str): The directory that the workspace exists in
	80	+ # dictionary (dict): The dict to generate a WorkspaceLocal from
	81	+ #
	82	+ # Returns:
	83	+ # (WorkspaceLocal): A newly instantiated WorkspaceLocal
	84	+ @classmethod
	85	+ def from_dict(cls, directory, dictionary):
	86	+ # Only know how to handle one format-version at the moment.
	87	+ format_version = int(dictionary['format-version'])
	88	+ assert format_version == BST_WORKSPACE_LOCAL_FORMAT_VERSION, \
	89	+ "Format version {} not found in {}".format(BST_WORKSPACE_LOCAL_FORMAT_VERSION, dictionary)
	90	+
	91	+ workspace_local = cls(directory)
	92	+ for item in dictionary['projects']:
	93	+ workspace_local._add_project(item['project-path'], item['element-name'])
	94	+
	95	+ return workspace_local
	96	+
	97	+ # create()
	98	+ #
	99	+ # Creates a new WorkspaceLocal
	100	+ #
	101	+ # Args:
	102	+ # directory (str): The directory that the workspace exists in
	103	+ # project_path (str): The path to the project to store
	104	+ # element_name (str): The name of the element within the project
	105	+ #
	106	+ # Returns:
	107	+ # (WorkspaceLocal): The created WorkspaceLocal
	108	+ @classmethod
	109	+ def create(cls, directory, project_path, element_name):
	110	+ # TODO: Load WorkspaceLocal if it exists, and maybe add project_path to it
	111	+ return cls(directory, project_path, element_name)
	112	+
	113	+ # load()
	114	+ #
	115	+ # Loads the WorkspaceLocal for a given directory. This directory may be a
	116	+ # subdirectory of the workspace's directory.
	117	+ #
	118	+ # Args:
	119	+ # directory (str): The directory
	120	+ # Returns:
	121	+ # (WorkspaceLocal): The created WorkspaceLocal, if in a workspace, or
	122	+ # (NoneType): None, if the directory is not inside a workspace.
	123	+ @classmethod
	124	+ def load(cls, directory):
	125	+ local_dir = cls.search_for_dir(directory)
	126	+ if local_dir:
	127	+ workspace_file = os.path.join(local_dir, WORKSPACE_LOCAL_FILE)
	128	+ data_dict = _yaml.load(workspace_file)
	129	+ return cls.from_dict(local_dir, data_dict)
	130	+ else:
	131	+ return None
	132	+
	133	+ # write()
	134	+ #
	135	+ # Writes the WorkspaceLocal to disk
	136	+ def write(self):
	137	+ os.makedirs(self._directory, exist_ok=True)
	138	+ _yaml.dump(self.to_dict(), self._get_filename())
	139	+
	140	+ # delete()
	141	+ #
	142	+ # Deletes the WorkspaceLocal from disk, if it exists.
	143	+ def delete(self):
	144	+ try:
	145	+ os.unlink(self._get_filename())
	146	+ except FileNotFoundError:
	147	+ pass
	148	+
	149	+ # search_for_dir()
	150	+ #
	151	+ # Returns the directory that contains the workspace local file,
	152	+ # searching upwards from search_dir.
	153	+ @staticmethod
	154	+ def search_for_dir(search_dir):
	155	+ return utils._search_upward_for_file(search_dir, WORKSPACE_LOCAL_FILE)
	156	+
	157	+ def _get_filename(self):
	158	+ return os.path.join(self._directory, WORKSPACE_LOCAL_FILE)
	159	+
	160	+ def _add_project(self, project_path, element_name):
	161	+ self._projects.append({'project-path': project_path, 'element-name': element_name})
28	162
29	163
30	164	# Workspace()

buildstream/element.py

---

...	...	@@ -2180,6 +2180,7 @@ class Element(Plugin):
2180	2180	stderr=stderr,
2181	2181	config=config,
2182	2182	server_url=self.__remote_execution_url,
	2183	+ bare_directory=bare_directory,
2183	2184	allow_real_directory=False)
2184	2185	yield sandbox
2185	2186

buildstream/utils.py

---

...	...	@@ -1198,3 +1198,17 @@ def _deduplicate(iterable, key=None):
1198	1198	def _get_link_mtime(path):
1199	1199	path_stat = os.lstat(path)
1200	1200	return path_stat.st_mtime
	1201	+
	1202	+
	1203	+# Returns the first directory to contain filename, or an empty string if
	1204	+# none found
	1205	+#
	1206	+def _search_upward_for_file(directory, filename):
	1207	+ directory = os.path.abspath(directory)
	1208	+ while not os.path.isfile(os.path.join(directory, filename)):
	1209	+ parent_dir = os.path.dirname(directory)
	1210	+ if directory == parent_dir:
	1211	+ return ""
	1212	+ directory = parent_dir
	1213	+
	1214	+ return directory

tests/frontend/workspace.py

---

...	...	@@ -93,6 +93,13 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
93	93
94	94	result.assert_success()
95	95
	96	+ # Assert that a .bstproject.yaml file has been created
	97	+ # and contains the path to the project
	98	+ bstproject_path = os.path.join(workspace_dir, '.bstproject.yaml')
	99	+ assert os.path.exists(bstproject_path)
	100	+ with open(bstproject_path) as f:
	101	+ assert project_path in f.read()
	102	+
96	103	# Assert that we are now buildable because the source is
97	104	# now cached.
98	105	assert cli.get_element_state(project_path, element_name) == 'buildable'
...	...	@@ -148,6 +155,10 @@ def test_open_force(cli, tmpdir, datafiles, kind):
148	155	# Assert the workspace dir still exists
149	156	assert os.path.exists(workspace)
150	157
	158	+ # Assert the bstproject doesn't exist
	159	+ bstproject_path = os.path.join(workspace, '.bstproject.yaml')
	160	+ assert not os.path.exists(bstproject_path)
	161	+
151	162	# Now open the workspace again with --force, this should happily succeed
152	163	result = cli.run(project=project, args=[
153	164	'workspace', 'open', '--force', element_name, workspace
...	...	@@ -436,7 +447,9 @@ def test_list(cli, tmpdir, datafiles):
436	447	@pytest.mark.datafiles(DATA_DIR)
437	448	@pytest.mark.parametrize("kind", repo_kinds)
438	449	@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
439		-def test_build(cli, tmpdir, datafiles, kind, strict):
	450	+@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
	451	+def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
	452	+ tmpdir = tmpdir_factory.mktemp('')
440	453	element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
441	454	checkout = os.path.join(str(tmpdir), 'checkout')
442	455
...	...	@@ -461,7 +474,8 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
461	474	# Build modified workspace
462	475	assert cli.get_element_state(project, element_name) == 'buildable'
463	476	assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
464		- result = cli.run(project=project, args=['build', element_name])
	477	+ result = cli.run(project=project if call_from == "project" else workspace,
	478	+ args=['build', element_name])
465	479	result.assert_success()
466	480	assert cli.get_element_state(project, element_name) == 'cached'
467	481	assert cli.get_element_key(project, element_name) != "{:?<64}".format('')

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.