[Notes] [Git][BuildStream/buildstream][master] 10 commits: utils.py: Add a helper for searching upwards for files

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch master at BuildStream / buildstream

Commits:

• 7892287a
  by Jonathan Maw at 2018-12-11T12:56:32Z

  utils.py: Add a helper for searching upwards for files
  
  i.e. with a given directory and filename, check parent directories until
  either a directory with the filename is found, or you reach the root of
  the filesystem.
  
  This is a part of #222
  

• 67c7a58d
  by Jonathan Maw at 2018-12-11T12:56:32Z

  Create and store data inside projects when opening workspaces
  
  Changes to _context.py:
  * Context has been extended to contain a WorkspaceProjectCache, as there
    are times when we want to use it before a Workspaces can be initialised
    (looking up a WorkspaceProject to find the directory that the project is
    in)
  
  Changes to _stream.py:
  * Removed staging the elements from workspace_open() and workspace_reset()
  
  Changes in _workspaces.py:
  * A new WorkspaceProject contains all the information needed to refer back
    to a project from its workspace (currently this is the project path and
    the element used to create this workspace)
  * This is stored within a new WorkspaceProjectCache object, which keeps
    WorkspaceProjects around so they don't need to be loaded from disk
    repeatedly.
  * Workspaces has been extended to contain the WorkspaceProjectCache, and
    will use it when opening and closing workspaces.
  * Workspaces.create_workspace has been extended to handle the staging of
    the element into the workspace, in addition to creating the equivalent
    WorkspaceProject file.
  
  This is a part of #222
  

• 64836b18
  by Jonathan Maw at 2018-12-11T12:56:32Z

  _project.py: Rename _ensure_project_dir to _find_project_dir
  
  This is a part of #222
  

• 7deeb2c3
  by Jonathan Maw at 2018-12-11T12:56:32Z

  cli.py: Use utils' search upwards helper when searching for project.conf
  
  This is a part of #222
  

• 496f0ab7
  by Jonathan Maw at 2018-12-11T12:56:32Z

  _project.py: Find project from workspace if outside of a project
  
  This is a part of #222
  

• 921f2bcb
  by Jonathan Maw at 2018-12-11T12:56:32Z

  _workspaces.py: Do not include .bstproject.yaml in the cache key
  
  This is a part of #222
  

• f145a3e4
  by Jonathan Maw at 2018-12-11T12:56:32Z

  cli: Interactively warn if the user is trying to close the workspace they're using to load the project
  
  This involves changes in:
  * _stream.py:
    * Add the helper Stream.workspace_is_required()
  * userconfig.yaml:
    * Add a default value for prompt.really-workspace-close-project-inaccessible
  * _context.py:
    * Load the prompt 'really-workspace-close-project-inaccessible' from
      user config.
  * cli.py:
    * If buildstream is invoked interactively, prompt the user to confirm
      that they want to close the workspace they're using to load this
      project.
  
  This is a part of #222
  

• 494cb7c6
  by Jonathan Maw at 2018-12-11T14:12:54Z

  tests: Test bst commands from an external workspace
  
  This is a part of #222
  

• fd1c5c5a
  by Jonathan Maw at 2018-12-11T14:12:54Z

  NEWS: Add an entry for being able to run commands from a workspace
  
  This is a part of #222
  

• 4219a6b4
  by Jonathan Maw at 2018-12-11T16:00:55Z

  Merge branch 'jonathan/workspace-fragment-create' into 'master'
  
  Support invoking buildstream from a workspace outside a project
  
  See merge request BuildStream/buildstream!924

11 changed files:

• NEWS
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_project.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/data/userconfig.yaml
• buildstream/utils.py
• tests/frontend/cross_junction_workspace.py
• tests/frontend/workspace.py
• tests/integration/shell.py

Changes:

NEWS

---

...	...	@@ -83,6 +83,10 @@ buildstream 1.3.1
83	83	plugin has now a tag tracking feature instead. This can be enabled
84	84	by setting 'track-tags'.
85	85
	86	+ o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
	87	+ commands to be run from a workspace that is not inside a project.
	88	+
	89	+
86	90	=================
87	91	buildstream 1.1.5
88	92	=================

buildstream/_context.py

---

...	...	@@ -32,7 +32,7 @@ from ._message import Message, MessageType
32	32	from ._profile import Topics, profile_start, profile_end
33	33	from ._artifactcache import ArtifactCache
34	34	from ._artifactcache.cascache import CASCache
35		-from ._workspaces import Workspaces
	35	+from ._workspaces import Workspaces, WorkspaceProjectCache
36	36	from .plugin import _plugin_lookup
37	37
38	38
...	...	@@ -122,6 +122,10 @@ class Context():
122	122	# remove a workspace directory.
123	123	self.prompt_workspace_close_remove_dir = None
124	124
	125	+ # Boolean, whether we double-check with the user that they meant to
	126	+ # close the workspace when they're using it to access the project.
	127	+ self.prompt_workspace_close_project_inaccessible = None
	128	+
125	129	# Boolean, whether we double-check with the user that they meant to do
126	130	# a hard reset of a workspace, potentially losing changes.
127	131	self.prompt_workspace_reset_hard = None
...	...	@@ -140,6 +144,7 @@ class Context():
140	144	self._projects = []
141	145	self._project_overrides = {}
142	146	self._workspaces = None
	147	+ self._workspace_project_cache = WorkspaceProjectCache()
143	148	self._log_handle = None
144	149	self._log_filename = None
145	150	self._cascache = None
...	...	@@ -250,12 +255,15 @@ class Context():
250	255	defaults, Mapping, 'prompt')
251	256	_yaml.node_validate(prompt, [
252	257	'auto-init', 'really-workspace-close-remove-dir',
	258	+ 'really-workspace-close-project-inaccessible',
253	259	'really-workspace-reset-hard',
254	260	])
255	261	self.prompt_auto_init = _node_get_option_str(
256	262	prompt, 'auto-init', ['ask', 'no']) == 'ask'
257	263	self.prompt_workspace_close_remove_dir = _node_get_option_str(
258	264	prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
	265	+ self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
	266	+ prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
259	267	self.prompt_workspace_reset_hard = _node_get_option_str(
260	268	prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
261	269
...	...	@@ -285,7 +293,7 @@ class Context():
285	293	#
286	294	def add_project(self, project):
287	295	if not self._projects:
288		- self._workspaces = Workspaces(project)
	296	+ self._workspaces = Workspaces(project, self._workspace_project_cache)
289	297	self._projects.append(project)
290	298
291	299	# get_projects():
...	...	@@ -312,6 +320,16 @@ class Context():
312	320	def get_workspaces(self):
313	321	return self._workspaces
314	322
	323	+ # get_workspace_project_cache():
	324	+ #
	325	+ # Return the WorkspaceProjectCache object used for this BuildStream invocation
	326	+ #
	327	+ # Returns:
	328	+ # (WorkspaceProjectCache): The WorkspaceProjectCache object
	329	+ #
	330	+ def get_workspace_project_cache(self):
	331	+ return self._workspace_project_cache
	332	+
315	333	# get_overrides():
316	334	#
317	335	# Fetch the override dictionary for the active project. This returns

buildstream/_frontend/cli.py

---

...	...	@@ -59,18 +59,9 @@ def complete_target(args, incomplete):
59	59	:return: all the possible user-specified completions for the param
60	60	"""
61	61
	62	+ from .. import utils
62	63	project_conf = 'project.conf'
63	64
64		- def ensure_project_dir(directory):
65		- directory = os.path.abspath(directory)
66		- while not os.path.isfile(os.path.join(directory, project_conf)):
67		- parent_dir = os.path.dirname(directory)
68		- if directory == parent_dir:
69		- break
70		- directory = parent_dir
71		-
72		- return directory
73		-
74	65	# First resolve the directory, in case there is an
75	66	# active --directory/-C option
76	67	#
...	...	@@ -89,7 +80,7 @@ def complete_target(args, incomplete):
89	80	else:
90	81	# Check if this directory or any of its parent directories
91	82	# contain a project config file
92		- base_directory = ensure_project_dir(base_directory)
	83	+ base_directory, _ = utils._search_upward_for_files(base_directory, [project_conf])
93	84
94	85	# Now parse the project.conf just to find the element path,
95	86	# this is unfortunately a bit heavy.
...	...	@@ -772,11 +763,19 @@ def workspace_close(app, remove_dir, all_, elements):
772	763
773	764	elements = app.stream.redirect_element_names(elements)
774	765
775		- # Check that the workspaces in question exist
	766	+ # Check that the workspaces in question exist, and that it's safe to
	767	+ # remove them.
776	768	nonexisting = []
777	769	for element_name in elements:
778	770	if not app.stream.workspace_exists(element_name):
779	771	nonexisting.append(element_name)
	772	+ if (app.stream.workspace_is_required(element_name) and app.interactive and
	773	+ app.context.prompt_workspace_close_project_inaccessible):
	774	+ click.echo("Removing '{}' will prevent you from running "
	775	+ "BuildStream commands from the current directory".format(element_name))
	776	+ if not click.confirm('Are you sure you want to close this workspace?'):
	777	+ click.echo('Aborting', err=True)
	778	+ sys.exit(-1)
780	779	if nonexisting:
781	780	raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
782	781

buildstream/_project.py

---

...	...	@@ -41,6 +41,7 @@ from .element import Element
41	41	from ._message import Message, MessageType
42	42	from ._includes import Includes
43	43	from ._platform import Platform
	44	+from ._workspaces import WORKSPACE_PROJECT_FILE
44	45
45	46
46	47	# Project Configuration file
...	...	@@ -95,8 +96,10 @@ class Project():
95	96	# The project name
96	97	self.name = None
97	98
98		- # The project directory
99		- self.directory = self._ensure_project_dir(directory)
	99	+ self._context = context # The invocation Context, a private member
	100	+
	101	+ # The project directory, and whether the element whose workspace it was invoked from
	102	+ self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
100	103
101	104	# Absolute path to where elements are loaded from within the project
102	105	self.element_path = None
...	...	@@ -117,7 +120,6 @@ class Project():
117	120	#
118	121	# Private Members
119	122	#
120		- self._context = context # The invocation Context
121	123
122	124	self._default_mirror = default_mirror # The name of the preferred mirror.
123	125
...	...	@@ -371,6 +373,14 @@ class Project():
371	373
372	374	self._load_second_pass()
373	375
	376	+ # invoked_from_workspace_element()
	377	+ #
	378	+ # Returns the element whose workspace was used to invoke buildstream
	379	+ # if buildstream was invoked from an external workspace
	380	+ #
	381	+ def invoked_from_workspace_element(self):
	382	+ return self._invoked_from_workspace_element
	383	+
374	384	# cleanup()
375	385	#
376	386	# Cleans up resources used loading elements
...	...	@@ -650,7 +660,7 @@ class Project():
650	660	# Source url aliases
651	661	output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
652	662
653		- # _ensure_project_dir()
	663	+ # _find_project_dir()
654	664	#
655	665	# Returns path of the project directory, if a configuration file is found
656	666	# in given directory or any of its parent directories.
...	...	@@ -661,18 +671,30 @@ class Project():
661	671	# Raises:
662	672	# LoadError if project.conf is not found
663	673	#
664		- def _ensure_project_dir(self, directory):
665		- directory = os.path.abspath(directory)
666		- while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
667		- parent_dir = os.path.dirname(directory)
668		- if directory == parent_dir:
669		- raise LoadError(
670		- LoadErrorReason.MISSING_PROJECT_CONF,
671		- '{} not found in current directory or any of its parent directories'
672		- .format(_PROJECT_CONF_FILE))
673		- directory = parent_dir
	674	+ # Returns:
	675	+ # (str) - the directory that contains the project, and
	676	+ # (str) - the name of the element required to find the project, or None
	677	+ #
	678	+ def _find_project_dir(self, directory):
	679	+ workspace_element = None
	680	+ found_directory, filename = utils._search_upward_for_files(
	681	+ directory, [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE]
	682	+ )
	683	+ if filename == _PROJECT_CONF_FILE:
	684	+ project_directory = found_directory
	685	+ elif filename == WORKSPACE_PROJECT_FILE:
	686	+ workspace_project_cache = self._context.get_workspace_project_cache()
	687	+ workspace_project = workspace_project_cache.get(found_directory)
	688	+ if workspace_project:
	689	+ project_directory = workspace_project.get_default_project_path()
	690	+ workspace_element = workspace_project.get_default_element()
	691	+ else:
	692	+ raise LoadError(
	693	+ LoadErrorReason.MISSING_PROJECT_CONF,
	694	+ '{} not found in current directory or any of its parent directories'
	695	+ .format(_PROJECT_CONF_FILE))
674	696
675		- return directory
	697	+ return project_directory, workspace_element
676	698
677	699	def _load_plugin_factories(self, config, output):
678	700	plugin_source_origins = [] # Origins of custom sources

buildstream/_stream.py

---

...	...	@@ -581,15 +581,7 @@ class Stream():
581	581	todo_elements = "\nDid not try to create workspaces for " + todo_elements
582	582	raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
583	583
584		- workspaces.create_workspace(target._get_full_name(), directory)
585		-
586		- if not no_checkout:
587		- with target.timed_activity("Staging sources to {}".format(directory)):
588		- target._open_workspace()
589		-
590		- # Saving the workspace once it is set up means that if the next workspace fails to be created before
591		- # the configuration gets saved. The successfully created workspace still gets saved.
592		- workspaces.save_config()
	584	+ workspaces.create_workspace(target, directory, checkout=not no_checkout)
593	585	self._message(MessageType.INFO, "Created a workspace for element: {}"
594	586	.format(target._get_full_name()))
595	587
...	...	@@ -672,10 +664,7 @@ class Stream():
672	664	.format(workspace_path, e)) from e
673	665
674	666	workspaces.delete_workspace(element._get_full_name())
675		- workspaces.create_workspace(element._get_full_name(), workspace_path)
676		-
677		- with element.timed_activity("Staging sources to {}".format(workspace_path)):
678		- element._open_workspace()
	667	+ workspaces.create_workspace(element, workspace_path, checkout=True)
679	668
680	669	self._message(MessageType.INFO,
681	670	"Reset workspace for {} at: {}".format(element.name,
...	...	@@ -707,6 +696,20 @@ class Stream():
707	696
708	697	return False
709	698
	699	+ # workspace_is_required()
	700	+ #
	701	+ # Checks whether the workspace belonging to element_name is required to
	702	+ # load the project
	703	+ #
	704	+ # Args:
	705	+ # element_name (str): The element whose workspace may be required
	706	+ #
	707	+ # Returns:
	708	+ # (bool): True if the workspace is required
	709	+ def workspace_is_required(self, element_name):
	710	+ invoked_elm = self._project.invoked_from_workspace_element()
	711	+ return invoked_elm == element_name
	712	+
710	713	# workspace_list
711	714	#
712	715	# Serializes the workspaces and dumps them in YAML to stdout.

buildstream/_workspaces.py

---

...	...	@@ -25,6 +25,202 @@ from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27	27	BST_WORKSPACE_FORMAT_VERSION = 3
	28	+BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1
	29	+WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
	30	+
	31	+
	32	+# WorkspaceProject()
	33	+#
	34	+# An object to contain various helper functions and data required for
	35	+# referring from a workspace back to buildstream.
	36	+#
	37	+# Args:
	38	+# directory (str): The directory that the workspace exists in.
	39	+#
	40	+class WorkspaceProject():
	41	+ def __init__(self, directory):
	42	+ self._projects = []
	43	+ self._directory = directory
	44	+
	45	+ # get_default_project_path()
	46	+ #
	47	+ # Retrieves the default path to a project.
	48	+ #
	49	+ # Returns:
	50	+ # (str): The path to a project
	51	+ #
	52	+ def get_default_project_path(self):
	53	+ return self._projects[0]['project-path']
	54	+
	55	+ # get_default_element()
	56	+ #
	57	+ # Retrieves the name of the element that owns this workspace.
	58	+ #
	59	+ # Returns:
	60	+ # (str): The name of an element
	61	+ #
	62	+ def get_default_element(self):
	63	+ return self._projects[0]['element-name']
	64	+
	65	+ # to_dict()
	66	+ #
	67	+ # Turn the members data into a dict for serialization purposes
	68	+ #
	69	+ # Returns:
	70	+ # (dict): A dict representation of the WorkspaceProject
	71	+ #
	72	+ def to_dict(self):
	73	+ ret = {
	74	+ 'projects': self._projects,
	75	+ 'format-version': BST_WORKSPACE_PROJECT_FORMAT_VERSION,
	76	+ }
	77	+ return ret
	78	+
	79	+ # from_dict()
	80	+ #
	81	+ # Loads a new WorkspaceProject from a simple dictionary
	82	+ #
	83	+ # Args:
	84	+ # directory (str): The directory that the workspace exists in
	85	+ # dictionary (dict): The dict to generate a WorkspaceProject from
	86	+ #
	87	+ # Returns:
	88	+ # (WorkspaceProject): A newly instantiated WorkspaceProject
	89	+ #
	90	+ @classmethod
	91	+ def from_dict(cls, directory, dictionary):
	92	+ # Only know how to handle one format-version at the moment.
	93	+ format_version = int(dictionary['format-version'])
	94	+ assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, \
	95	+ "Format version {} not found in {}".format(BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary)
	96	+
	97	+ workspace_project = cls(directory)
	98	+ for item in dictionary['projects']:
	99	+ workspace_project.add_project(item['project-path'], item['element-name'])
	100	+
	101	+ return workspace_project
	102	+
	103	+ # load()
	104	+ #
	105	+ # Loads the WorkspaceProject for a given directory.
	106	+ #
	107	+ # Args:
	108	+ # directory (str): The directory
	109	+ # Returns:
	110	+ # (WorkspaceProject): The created WorkspaceProject, if in a workspace, or
	111	+ # (NoneType): None, if the directory is not inside a workspace.
	112	+ #
	113	+ @classmethod
	114	+ def load(cls, directory):
	115	+ workspace_file = os.path.join(directory, WORKSPACE_PROJECT_FILE)
	116	+ if os.path.exists(workspace_file):
	117	+ data_dict = _yaml.load(workspace_file)
	118	+ return cls.from_dict(directory, data_dict)
	119	+ else:
	120	+ return None
	121	+
	122	+ # write()
	123	+ #
	124	+ # Writes the WorkspaceProject to disk
	125	+ #
	126	+ def write(self):
	127	+ os.makedirs(self._directory, exist_ok=True)
	128	+ _yaml.dump(self.to_dict(), self.get_filename())
	129	+
	130	+ # get_filename()
	131	+ #
	132	+ # Returns the full path to the workspace local project file
	133	+ #
	134	+ def get_filename(self):
	135	+ return os.path.join(self._directory, WORKSPACE_PROJECT_FILE)
	136	+
	137	+ # add_project()
	138	+ #
	139	+ # Adds an entry containing the project's path and element's name.
	140	+ #
	141	+ # Args:
	142	+ # project_path (str): The path to the project that opened the workspace.
	143	+ # element_name (str): The name of the element that the workspace belongs to.
	144	+ #
	145	+ def add_project(self, project_path, element_name):
	146	+ assert (project_path and element_name)
	147	+ self._projects.append({'project-path': project_path, 'element-name': element_name})
	148	+
	149	+
	150	+# WorkspaceProjectCache()
	151	+#
	152	+# A class to manage workspace project data for multiple workspaces.
	153	+#
	154	+class WorkspaceProjectCache():
	155	+ def __init__(self):
	156	+ self._projects = {} # Mapping of a workspace directory to its WorkspaceProject
	157	+
	158	+ # get()
	159	+ #
	160	+ # Returns a WorkspaceProject for a given directory, retrieving from the cache if
	161	+ # present.
	162	+ #
	163	+ # Args:
	164	+ # directory (str): The directory to search for a WorkspaceProject.
	165	+ #
	166	+ # Returns:
	167	+ # (WorkspaceProject): The WorkspaceProject that was found for that directory.
	168	+ # or (NoneType): None, if no WorkspaceProject can be found.
	169	+ #
	170	+ def get(self, directory):
	171	+ try:
	172	+ workspace_project = self._projects[directory]
	173	+ except KeyError:
	174	+ workspace_project = WorkspaceProject.load(directory)
	175	+ if workspace_project:
	176	+ self._projects[directory] = workspace_project
	177	+
	178	+ return workspace_project
	179	+
	180	+ # add()
	181	+ #
	182	+ # Adds the project path and element name to the WorkspaceProject that exists
	183	+ # for that directory
	184	+ #
	185	+ # Args:
	186	+ # directory (str): The directory to search for a WorkspaceProject.
	187	+ # project_path (str): The path to the project that refers to this workspace
	188	+ # element_name (str): The element in the project that was refers to this workspace
	189	+ #
	190	+ # Returns:
	191	+ # (WorkspaceProject): The WorkspaceProject that was found for that directory.
	192	+ #
	193	+ def add(self, directory, project_path, element_name):
	194	+ workspace_project = self.get(directory)
	195	+ if not workspace_project:
	196	+ workspace_project = WorkspaceProject(directory)
	197	+ self._projects[directory] = workspace_project
	198	+
	199	+ workspace_project.add_project(project_path, element_name)
	200	+ return workspace_project
	201	+
	202	+ # remove()
	203	+ #
	204	+ # Removes the project path and element name from the WorkspaceProject that exists
	205	+ # for that directory.
	206	+ #
	207	+ # NOTE: This currently just deletes the file, but with support for multiple
	208	+ # projects opening the same workspace, this will involve decreasing the count
	209	+ # and deleting the file if there are no more projects.
	210	+ #
	211	+ # Args:
	212	+ # directory (str): The directory to search for a WorkspaceProject.
	213	+ #
	214	+ def remove(self, directory):
	215	+ workspace_project = self.get(directory)
	216	+ if not workspace_project:
	217	+ raise LoadError(LoadErrorReason.MISSING_FILE,
	218	+ "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE))
	219	+ path = workspace_project.get_filename()
	220	+ try:
	221	+ os.unlink(path)
	222	+ except FileNotFoundError:
	223	+ pass
28	224
29	225
30	226	# Workspace()
...	...	@@ -174,10 +370,15 @@ class Workspace():
174	370	if recalculate or self._key is None:
175	371	fullpath = self.get_absolute_path()
176	372
	373	+ excluded_files = (WORKSPACE_PROJECT_FILE,)
	374	+
177	375	# Get a list of tuples of the the project relative paths and fullpaths
178	376	if os.path.isdir(fullpath):
179	377	filelist = utils.list_relative_paths(fullpath)
180		- filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
	378	+ filelist = [
	379	+ (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
	380	+ if relpath not in excluded_files
	381	+ ]
181	382	else:
182	383	filelist = [(self.get_absolute_path(), fullpath)]
183	384
...	...	@@ -199,12 +400,14 @@ class Workspace():
199	400	#
200	401	# Args:
201	402	# toplevel_project (Project): Top project used to resolve paths.
	403	+# workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects
202	404	#
203	405	class Workspaces():
204		- def __init__(self, toplevel_project):
	406	+ def __init__(self, toplevel_project, workspace_project_cache):
205	407	self._toplevel_project = toplevel_project
206	408	self._bst_directory = os.path.join(toplevel_project.directory, ".bst")
207	409	self._workspaces = self._load_config()
	410	+ self._workspace_project_cache = workspace_project_cache
208	411
209	412	# list()
210	413	#
...	...	@@ -219,19 +422,36 @@ class Workspaces():
219	422
220	423	# create_workspace()
221	424	#
222		- # Create a workspace in the given path for the given element.
	425	+ # Create a workspace in the given path for the given element, and potentially
	426	+ # checks-out the target into it.
223	427	#
224	428	# Args:
225		- # element_name (str) - The element name to create a workspace for
	429	+ # target (Element) - The element to create a workspace for
226	430	# path (str) - The path in which the workspace should be kept
	431	+ # checkout (bool): Whether to check-out the element's sources into the directory
227	432	#
228		- def create_workspace(self, element_name, path):
229		- if path.startswith(self._toplevel_project.directory):
230		- path = os.path.relpath(path, self._toplevel_project.directory)
	433	+ def create_workspace(self, target, path, *, checkout):
	434	+ element_name = target._get_full_name()
	435	+ project_dir = self._toplevel_project.directory
	436	+ if path.startswith(project_dir):
	437	+ workspace_path = os.path.relpath(path, project_dir)
	438	+ else:
	439	+ workspace_path = path
231	440
232		- self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
	441	+ self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path)
233	442
234		- return self._workspaces[element_name]
	443	+ if checkout:
	444	+ with target.timed_activity("Staging sources to {}".format(path)):
	445	+ target._open_workspace()
	446	+
	447	+ workspace_project = self._workspace_project_cache.add(path, project_dir, element_name)
	448	+ project_file_path = workspace_project.get_filename()
	449	+
	450	+ if os.path.exists(project_file_path):
	451	+ target.warn("{} was staged from this element's sources".format(WORKSPACE_PROJECT_FILE))
	452	+ workspace_project.write()
	453	+
	454	+ self.save_config()
235	455
236	456	# get_workspace()
237	457	#
...	...	@@ -280,8 +500,19 @@ class Workspaces():
280	500	# element_name (str) - The element name whose workspace to delete
281	501	#
282	502	def delete_workspace(self, element_name):
	503	+ workspace = self.get_workspace(element_name)
283	504	del self._workspaces[element_name]
284	505
	506	+ # Remove from the cache if it exists
	507	+ try:
	508	+ self._workspace_project_cache.remove(workspace.get_absolute_path())
	509	+ except LoadError as e:
	510	+ # We might be closing a workspace with a deleted directory
	511	+ if e.reason == LoadErrorReason.MISSING_FILE:
	512	+ pass
	513	+ else:
	514	+ raise
	515	+
285	516	# save_config()
286	517	#
287	518	# Dump the current workspace element to the project configuration

buildstream/data/userconfig.yaml

---

...	...	@@ -128,6 +128,14 @@ prompt:
128	128	#
129	129	really-workspace-close-remove-dir: ask
130	130
	131	+ # Whether to really proceed with 'bst workspace close' when doing so would
	132	+ # stop them from running bst commands in this workspace.
	133	+ #
	134	+ # ask - Ask the user if they are sure.
	135	+ # yes - Always close, without asking.
	136	+ #
	137	+ really-workspace-close-project-inaccessible: ask
	138	+
131	139	# Whether to really proceed with 'bst workspace reset' doing a hard reset of
132	140	# a workspace, potentially losing changes.
133	141	#

buildstream/utils.py

---

...	...	@@ -1259,3 +1259,34 @@ def _message_digest(message_buffer):
1259	1259	digest.hash = sha.hexdigest()
1260	1260	digest.size_bytes = len(message_buffer)
1261	1261	return digest
	1262	+
	1263	+
	1264	+# _search_upward_for_files()
	1265	+#
	1266	+# Searches upwards (from directory, then directory's parent directory...)
	1267	+# for any of the files listed in `filenames`.
	1268	+#
	1269	+# If multiple filenames are specified, and present in the same directory,
	1270	+# the first filename in the list will be returned.
	1271	+#
	1272	+# Args:
	1273	+# directory (str): The directory to begin searching for files from
	1274	+# filenames (list of str): The names of files to search for
	1275	+#
	1276	+# Returns:
	1277	+# (str): The directory a file was found in, or None
	1278	+# (str): The name of the first file that was found in that directory, or None
	1279	+#
	1280	+def _search_upward_for_files(directory, filenames):
	1281	+ directory = os.path.abspath(directory)
	1282	+ while True:
	1283	+ for filename in filenames:
	1284	+ file_path = os.path.join(directory, filename)
	1285	+ if os.path.isfile(file_path):
	1286	+ return directory, filename
	1287	+
	1288	+ parent_dir = os.path.dirname(directory)
	1289	+ if directory == parent_dir:
	1290	+ # i.e. we've reached the root of the filesystem
	1291	+ return None, None
	1292	+ directory = parent_dir

tests/frontend/cross_junction_workspace.py

---

...	...	@@ -115,3 +115,23 @@ def test_close_all_cross_junction(cli, tmpdir):
115	115	assert isinstance(loaded.get('workspaces'), list)
116	116	workspaces = loaded['workspaces']
117	117	assert len(workspaces) == 0
	118	+
	119	+
	120	+def test_subdir_command_cross_junction(cli, tmpdir):
	121	+ # i.e. commands can be run successfully from a subdirectory of the
	122	+ # junction's workspace, in case project loading logic has gone wrong
	123	+ project = prepare_junction_project(cli, tmpdir)
	124	+ workspace = os.path.join(str(tmpdir), 'workspace')
	125	+ junction_element = 'sub.bst'
	126	+
	127	+ # Open the junction as a workspace
	128	+ args = ['workspace', 'open', '--directory', workspace, junction_element]
	129	+ result = cli.run(project=project, args=args)
	130	+ result.assert_success()
	131	+
	132	+ # Run commands from a subdirectory of the workspace
	133	+ newdir = os.path.join(str(workspace), "newdir")
	134	+ element_name = 'data.bst'
	135	+ os.makedirs(newdir)
	136	+ result = cli.run(project=str(workspace), args=['-C', newdir, 'show', element_name])
	137	+ result.assert_success()

tests/frontend/workspace.py

---

...	...	@@ -31,6 +31,7 @@ import shutil
31	31	import subprocess
32	32	from ruamel.yaml.comments import CommentedSet
33	33	from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
	34	+from tests.testutils import create_artifact_share
34	35
35	36	from buildstream import _yaml
36	37	from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
...	...	@@ -615,9 +616,12 @@ def test_list(cli, tmpdir, datafiles):
615	616	@pytest.mark.datafiles(DATA_DIR)
616	617	@pytest.mark.parametrize("kind", repo_kinds)
617	618	@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
618		-def test_build(cli, tmpdir, datafiles, kind, strict):
	619	+@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
	620	+def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
	621	+ tmpdir = tmpdir_factory.mktemp('')
619	622	element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
620	623	checkout = os.path.join(str(tmpdir), 'checkout')
	624	+ args_pre = ['-C', workspace] if call_from == "workspace" else []
621	625
622	626	# Modify workspace
623	627	shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
...	...	@@ -640,15 +644,14 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
640	644	# Build modified workspace
641	645	assert cli.get_element_state(project, element_name) == 'buildable'
642	646	assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
643		- result = cli.run(project=project, args=['build', element_name])
	647	+ result = cli.run(project=project, args=args_pre + ['build', element_name])
644	648	result.assert_success()
645	649	assert cli.get_element_state(project, element_name) == 'cached'
646	650	assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
647	651
648	652	# Checkout the result
649		- result = cli.run(project=project, args=[
650		- 'checkout', element_name, checkout
651		- ])
	653	+ result = cli.run(project=project,
	654	+ args=args_pre + ['checkout', element_name, checkout])
652	655	result.assert_success()
653	656
654	657	# Check that the pony.conf from the modified workspace exists
...	...	@@ -1055,3 +1058,137 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
1055	1058	result = cli.run(project=project, args=["build", element_name])
1056	1059	assert "BUG" not in result.stderr
1057	1060	assert cli.get_element_state(project, element_name) != "cached"
	1061	+
	1062	+
	1063	+@pytest.mark.datafiles(DATA_DIR)
	1064	+@pytest.mark.parametrize('subdir', [True, False], ids=["subdir", "no-subdir"])
	1065	+def test_external_fetch(cli, datafiles, tmpdir_factory, subdir):
	1066	+ # Fetching from a workspace outside a project doesn't fail horribly
	1067	+ tmpdir = tmpdir_factory.mktemp('')
	1068	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1069	+
	1070	+ if subdir:
	1071	+ call_dir = os.path.join(workspace, 'usr')
	1072	+ else:
	1073	+ call_dir = workspace
	1074	+
	1075	+ result = cli.run(project=project, args=['-C', call_dir, 'fetch', element_name])
	1076	+ result.assert_success()
	1077	+
	1078	+ # We already fetched it by opening the workspace, but we're also checking
	1079	+ # `bst show` works here
	1080	+ assert cli.get_element_state(project, element_name) == 'buildable'
	1081	+
	1082	+
	1083	+@pytest.mark.datafiles(DATA_DIR)
	1084	+def test_external_push_pull(cli, datafiles, tmpdir_factory):
	1085	+ # Pushing and pulling to/from an artifact cache works from an external workspace
	1086	+ tmpdir = tmpdir_factory.mktemp('')
	1087	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1088	+
	1089	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
	1090	+ result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
	1091	+ result.assert_success()
	1092	+
	1093	+ cli.configure({
	1094	+ 'artifacts': {'url': share.repo, 'push': True}
	1095	+ })
	1096	+
	1097	+ result = cli.run(project=project, args=['-C', workspace, 'push', element_name])
	1098	+ result.assert_success()
	1099	+
	1100	+ result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', element_name])
	1101	+ result.assert_success()
	1102	+
	1103	+
	1104	+@pytest.mark.datafiles(DATA_DIR)
	1105	+def test_external_track(cli, datafiles, tmpdir_factory):
	1106	+ # Tracking does not get horribly confused
	1107	+ tmpdir = tmpdir_factory.mktemp('')
	1108	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True)
	1109	+
	1110	+ # The workspace is necessarily already tracked, so we only care that
	1111	+ # there's no weird errors.
	1112	+ result = cli.run(project=project, args=['-C', workspace, 'track', element_name])
	1113	+ result.assert_success()
	1114	+
	1115	+
	1116	+@pytest.mark.datafiles(DATA_DIR)
	1117	+def test_external_open_other(cli, datafiles, tmpdir_factory):
	1118	+ # >From inside an external workspace, open another workspace
	1119	+ tmpdir1 = tmpdir_factory.mktemp('')
	1120	+ tmpdir2 = tmpdir_factory.mktemp('')
	1121	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1122	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1123	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1124	+
	1125	+ # Closing the other element first, because I'm too lazy to create an
	1126	+ # element without opening it
	1127	+ result = cli.run(project=project, args=['workspace', 'close', beta_element])
	1128	+ result.assert_success()
	1129	+
	1130	+ result = cli.run(project=project, args=[
	1131	+ '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element
	1132	+ ])
	1133	+ result.assert_success()
	1134	+
	1135	+
	1136	+@pytest.mark.datafiles(DATA_DIR)
	1137	+def test_external_close_other(cli, datafiles, tmpdir_factory):
	1138	+ # >From inside an external workspace, close the other workspace
	1139	+ tmpdir1 = tmpdir_factory.mktemp('')
	1140	+ tmpdir2 = tmpdir_factory.mktemp('')
	1141	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1142	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1143	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1144	+
	1145	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
	1146	+ result.assert_success()
	1147	+
	1148	+
	1149	+@pytest.mark.datafiles(DATA_DIR)
	1150	+def test_external_close_self(cli, datafiles, tmpdir_factory):
	1151	+ # >From inside an external workspace, close it
	1152	+ tmpdir1 = tmpdir_factory.mktemp('')
	1153	+ tmpdir2 = tmpdir_factory.mktemp('')
	1154	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1155	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1156	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1157	+
	1158	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', alpha_element])
	1159	+ result.assert_success()
	1160	+
	1161	+
	1162	+@pytest.mark.datafiles(DATA_DIR)
	1163	+def test_external_reset_other(cli, datafiles, tmpdir_factory):
	1164	+ tmpdir1 = tmpdir_factory.mktemp('')
	1165	+ tmpdir2 = tmpdir_factory.mktemp('')
	1166	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1167	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1168	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1169	+
	1170	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element])
	1171	+ result.assert_success()
	1172	+
	1173	+
	1174	+@pytest.mark.datafiles(DATA_DIR)
	1175	+def test_external_reset_self(cli, datafiles, tmpdir):
	1176	+ element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1177	+
	1178	+ # Command succeeds
	1179	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', element])
	1180	+ result.assert_success()
	1181	+
	1182	+ # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted)
	1183	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
	1184	+ result.assert_success()
	1185	+
	1186	+
	1187	+@pytest.mark.datafiles(DATA_DIR)
	1188	+def test_external_list(cli, datafiles, tmpdir_factory):
	1189	+ tmpdir = tmpdir_factory.mktemp('')
	1190	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1191	+ element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1192	+
	1193	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
	1194	+ result.assert_success()

tests/integration/shell.py

---

...	...	@@ -353,3 +353,29 @@ def test_integration_devices(cli, tmpdir, datafiles):
353	353
354	354	result = execute_shell(cli, project, ["true"], element=element_name)
355	355	assert result.exit_code == 0
	356	+
	357	+
	358	+# Test that a shell can be opened from an external workspace
	359	+@pytest.mark.datafiles(DATA_DIR)
	360	+@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
	361	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	362	+def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell):
	363	+ tmpdir = tmpdir_factory.mktemp("")
	364	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	365	+ element_name = 'autotools/amhello.bst'
	366	+ workspace_dir = os.path.join(str(tmpdir), 'workspace')
	367	+
	368	+ result = cli.run(project=project, args=[
	369	+ 'workspace', 'open', '--directory', workspace_dir, element_name
	370	+ ])
	371	+ result.assert_success()
	372	+
	373	+ result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
	374	+ result.assert_success()
	375	+
	376	+ command = ['shell']
	377	+ if build_shell == 'build':
	378	+ command.append('--build')
	379	+ command.extend([element_name, '--', 'true'])
	380	+ result = cli.run(project=project, cwd=workspace_dir, args=command)
	381	+ result.assert_success()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.