[Notes] [Git][BuildStream/buildstream][jonathan/workspace-fragment-create] 5 commits: _project.py: Find project from workspace if outside of a project

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch jonathan/workspace-fragment-create at BuildStream / buildstream

Commits:

• 48edeea8
  by Jonathan Maw at 2018-12-11T11:08:58Z

  _project.py: Find project from workspace if outside of a project
  
  This is a part of #222
  

• 805f7869
  by Jonathan Maw at 2018-12-11T11:08:58Z

  _workspaces.py: Do not include .bstproject.yaml in the cache key
  
  This is a part of #222
  

• 48f3f0e7
  by Jonathan Maw at 2018-12-11T11:08:58Z

  cli: Interactively warn if the user is trying to close the workspace they're using to load the project
  
  This involves changes in:
  * _stream.py:
    * Add the helper Stream.workspace_is_required()
  * userconfig.yaml:
    * Add a default value for prompt.really-workspace-close-project-inaccessible
  * _context.py:
    * Load the prompt 'really-workspace-close-project-inaccessible' from
      user config.
  * cli.py:
    * If buildstream is invoked interactively, prompt the user to confirm
      that they want to close the workspace they're using to load this
      project.
  
  This is a part of #222
  

• a5493b0e
  by Jonathan Maw at 2018-12-11T11:08:58Z

  tests: Test bst commands from an external workspace
  
  This is a part of #222
  

• f3ca6f30
  by Jonathan Maw at 2018-12-11T11:08:59Z

  NEWS: Add an entry for being able to run commands from a workspace
  
  This is a part of #222
  

9 changed files:

• NEWS
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_project.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/data/userconfig.yaml
• tests/frontend/workspace.py
• tests/integration/shell.py

Changes:

NEWS

---

...	...	@@ -83,6 +83,10 @@ buildstream 1.3.1
83	83	plugin has now a tag tracking feature instead. This can be enabled
84	84	by setting 'track-tags'.
85	85
	86	+ o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
	87	+ commands to be run from a workspace that is not inside a project.
	88	+
	89	+
86	90	=================
87	91	buildstream 1.1.5
88	92	=================

buildstream/_context.py

---

...	...	@@ -122,6 +122,10 @@ class Context():
122	122	# remove a workspace directory.
123	123	self.prompt_workspace_close_remove_dir = None
124	124
	125	+ # Boolean, whether we double-check with the user that they meant to
	126	+ # close the workspace when they're using it to access the project.
	127	+ self.prompt_workspace_close_project_inaccessible = None
	128	+
125	129	# Boolean, whether we double-check with the user that they meant to do
126	130	# a hard reset of a workspace, potentially losing changes.
127	131	self.prompt_workspace_reset_hard = None
...	...	@@ -251,12 +255,15 @@ class Context():
251	255	defaults, Mapping, 'prompt')
252	256	_yaml.node_validate(prompt, [
253	257	'auto-init', 'really-workspace-close-remove-dir',
	258	+ 'really-workspace-close-project-inaccessible',
254	259	'really-workspace-reset-hard',
255	260	])
256	261	self.prompt_auto_init = _node_get_option_str(
257	262	prompt, 'auto-init', ['ask', 'no']) == 'ask'
258	263	self.prompt_workspace_close_remove_dir = _node_get_option_str(
259	264	prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
	265	+ self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
	266	+ prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
260	267	self.prompt_workspace_reset_hard = _node_get_option_str(
261	268	prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
262	269

buildstream/_frontend/cli.py

---

...	...	@@ -763,11 +763,19 @@ def workspace_close(app, remove_dir, all_, elements):
763	763
764	764	elements = app.stream.redirect_element_names(elements)
765	765
766		- # Check that the workspaces in question exist
	766	+ # Check that the workspaces in question exist, and that it's safe to
	767	+ # remove them.
767	768	nonexisting = []
768	769	for element_name in elements:
769	770	if not app.stream.workspace_exists(element_name):
770	771	nonexisting.append(element_name)
	772	+ if (app.stream.workspace_is_required(element_name) and app.interactive and
	773	+ app.context.prompt_workspace_close_project_inaccessible):
	774	+ click.echo("Removing '{}' will prevent you from running "
	775	+ "BuildStream commands from the current directory".format(element_name))
	776	+ if not click.confirm('Are you sure you want to close this workspace?'):
	777	+ click.echo('Aborting', err=True)
	778	+ sys.exit(-1)
771	779	if nonexisting:
772	780	raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
773	781

buildstream/_project.py

---

...	...	@@ -41,6 +41,7 @@ from .element import Element
41	41	from ._message import Message, MessageType
42	42	from ._includes import Includes
43	43	from ._platform import Platform
	44	+from ._workspaces import WORKSPACE_PROJECT_FILE
44	45
45	46
46	47	# Project Configuration file
...	...	@@ -95,8 +96,10 @@ class Project():
95	96	# The project name
96	97	self.name = None
97	98
98		- # The project directory
99		- self.directory = self._find_project_dir(directory)
	99	+ self._context = context # The invocation Context, a private member
	100	+
	101	+ # The project directory, and whether the element whose workspace it was invoked from
	102	+ self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
100	103
101	104	# Absolute path to where elements are loaded from within the project
102	105	self.element_path = None
...	...	@@ -117,7 +120,6 @@ class Project():
117	120	#
118	121	# Private Members
119	122	#
120		- self._context = context # The invocation Context
121	123
122	124	self._default_mirror = default_mirror # The name of the preferred mirror.
123	125
...	...	@@ -371,6 +373,14 @@ class Project():
371	373
372	374	self._load_second_pass()
373	375
	376	+ # invoked_from_workspace_element()
	377	+ #
	378	+ # Returns the element whose workspace was used to invoke buildstream
	379	+ # if buildstream was invoked from an external workspace
	380	+ #
	381	+ def invoked_from_workspace_element(self):
	382	+ return self._invoked_from_workspace_element
	383	+
374	384	# cleanup()
375	385	#
376	386	# Cleans up resources used loading elements
...	...	@@ -661,18 +671,30 @@ class Project():
661	671	# Raises:
662	672	# LoadError if project.conf is not found
663	673	#
	674	+ # Returns:
	675	+ # (str) - the directory that contains the project, and
	676	+ # (str) - the name of the element required to find the project, or None
	677	+ #
664	678	def _find_project_dir(self, directory):
665		- directory = os.path.abspath(directory)
666		- while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
667		- parent_dir = os.path.dirname(directory)
668		- if directory == parent_dir:
669		- raise LoadError(
670		- LoadErrorReason.MISSING_PROJECT_CONF,
671		- '{} not found in current directory or any of its parent directories'
672		- .format(_PROJECT_CONF_FILE))
673		- directory = parent_dir
	679	+ workspace_element = None
	680	+ found_directory, filename = utils._search_upward_for_files(
	681	+ directory, [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE]
	682	+ )
	683	+ if filename == _PROJECT_CONF_FILE:
	684	+ project_directory = found_directory
	685	+ elif filename == WORKSPACE_PROJECT_FILE:
	686	+ workspace_project_cache = self._context.get_workspace_project_cache()
	687	+ workspace_project = workspace_project_cache.get(found_directory)
	688	+ if workspace_project:
	689	+ project_directory = workspace_project.get_default_project_path()
	690	+ workspace_element = workspace_project.get_default_element()
	691	+ else:
	692	+ raise LoadError(
	693	+ LoadErrorReason.MISSING_PROJECT_CONF,
	694	+ '{} not found in current directory or any of its parent directories'
	695	+ .format(_PROJECT_CONF_FILE))
674	696
675		- return directory
	697	+ return project_directory, workspace_element
676	698
677	699	def _load_plugin_factories(self, config, output):
678	700	plugin_source_origins = [] # Origins of custom sources

buildstream/_stream.py

---

...	...	@@ -696,6 +696,20 @@ class Stream():
696	696
697	697	return False
698	698
	699	+ # workspace_is_required()
	700	+ #
	701	+ # Checks whether the workspace belonging to element_name is required to
	702	+ # load the project
	703	+ #
	704	+ # Args:
	705	+ # element_name (str): The element whose workspace may be required
	706	+ #
	707	+ # Returns:
	708	+ # (bool): True if the workspace is required
	709	+ def workspace_is_required(self, element_name):
	710	+ invoked_elm = self._project.invoked_from_workspace_element()
	711	+ return invoked_elm == element_name
	712	+
699	713	# workspace_list
700	714	#
701	715	# Serializes the workspaces and dumps them in YAML to stdout.

buildstream/_workspaces.py

---

...	...	@@ -370,10 +370,15 @@ class Workspace():
370	370	if recalculate or self._key is None:
371	371	fullpath = self.get_absolute_path()
372	372
	373	+ excluded_files = (WORKSPACE_PROJECT_FILE,)
	374	+
373	375	# Get a list of tuples of the the project relative paths and fullpaths
374	376	if os.path.isdir(fullpath):
375	377	filelist = utils.list_relative_paths(fullpath)
376		- filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
	378	+ filelist = [
	379	+ (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
	380	+ if relpath not in excluded_files
	381	+ ]
377	382	else:
378	383	filelist = [(self.get_absolute_path(), fullpath)]
379	384

buildstream/data/userconfig.yaml

---

...	...	@@ -128,6 +128,14 @@ prompt:
128	128	#
129	129	really-workspace-close-remove-dir: ask
130	130
	131	+ # Whether to really proceed with 'bst workspace close' when doing so would
	132	+ # stop them from running bst commands in this workspace.
	133	+ #
	134	+ # ask - Ask the user if they are sure.
	135	+ # yes - Always close, without asking.
	136	+ #
	137	+ really-workspace-close-project-inaccessible: ask
	138	+
131	139	# Whether to really proceed with 'bst workspace reset' doing a hard reset of
132	140	# a workspace, potentially losing changes.
133	141	#

tests/frontend/workspace.py

---

...	...	@@ -31,6 +31,7 @@ import shutil
31	31	import subprocess
32	32	from ruamel.yaml.comments import CommentedSet
33	33	from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
	34	+from tests.testutils import create_artifact_share
34	35
35	36	from buildstream import _yaml
36	37	from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
...	...	@@ -615,9 +616,12 @@ def test_list(cli, tmpdir, datafiles):
615	616	@pytest.mark.datafiles(DATA_DIR)
616	617	@pytest.mark.parametrize("kind", repo_kinds)
617	618	@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
618		-def test_build(cli, tmpdir, datafiles, kind, strict):
	619	+@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
	620	+def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
	621	+ tmpdir = tmpdir_factory.mktemp('')
619	622	element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
620	623	checkout = os.path.join(str(tmpdir), 'checkout')
	624	+ args_pre = ['-C', workspace] if call_from == "workspace" else []
621	625
622	626	# Modify workspace
623	627	shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
...	...	@@ -640,15 +644,14 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
640	644	# Build modified workspace
641	645	assert cli.get_element_state(project, element_name) == 'buildable'
642	646	assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
643		- result = cli.run(project=project, args=['build', element_name])
	647	+ result = cli.run(project=project, args=args_pre + ['build', element_name])
644	648	result.assert_success()
645	649	assert cli.get_element_state(project, element_name) == 'cached'
646	650	assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
647	651
648	652	# Checkout the result
649		- result = cli.run(project=project, args=[
650		- 'checkout', element_name, checkout
651		- ])
	653	+ result = cli.run(project=project,
	654	+ args=args_pre + ['checkout', element_name, checkout])
652	655	result.assert_success()
653	656
654	657	# Check that the pony.conf from the modified workspace exists
...	...	@@ -1055,3 +1058,131 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
1055	1058	result = cli.run(project=project, args=["build", element_name])
1056	1059	assert "BUG" not in result.stderr
1057	1060	assert cli.get_element_state(project, element_name) != "cached"
	1061	+
	1062	+
	1063	+@pytest.mark.datafiles(DATA_DIR)
	1064	+def test_external_fetch(cli, datafiles, tmpdir_factory):
	1065	+ # Fetching from a workspace outside a project doesn't fail horribly
	1066	+ tmpdir = tmpdir_factory.mktemp('')
	1067	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1068	+
	1069	+ result = cli.run(project=project, args=['-C', workspace, 'fetch', element_name])
	1070	+ result.assert_success()
	1071	+
	1072	+ # We already fetched it by opening the workspace, but we're also checking
	1073	+ # `bst show` works here
	1074	+ assert cli.get_element_state(project, element_name) == 'buildable'
	1075	+
	1076	+
	1077	+@pytest.mark.datafiles(DATA_DIR)
	1078	+def test_external_push_pull(cli, datafiles, tmpdir_factory):
	1079	+ # Pushing and pulling to/from an artifact cache works from an external workspace
	1080	+ tmpdir = tmpdir_factory.mktemp('')
	1081	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1082	+
	1083	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
	1084	+ result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
	1085	+ result.assert_success()
	1086	+
	1087	+ cli.configure({
	1088	+ 'artifacts': {'url': share.repo, 'push': True}
	1089	+ })
	1090	+
	1091	+ result = cli.run(project=project, args=['-C', workspace, 'push', element_name])
	1092	+ result.assert_success()
	1093	+
	1094	+ result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', element_name])
	1095	+ result.assert_success()
	1096	+
	1097	+
	1098	+@pytest.mark.datafiles(DATA_DIR)
	1099	+def test_external_track(cli, datafiles, tmpdir_factory):
	1100	+ # Tracking does not get horribly confused
	1101	+ tmpdir = tmpdir_factory.mktemp('')
	1102	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True)
	1103	+
	1104	+ # The workspace is necessarily already tracked, so we only care that
	1105	+ # there's no weird errors.
	1106	+ result = cli.run(project=project, args=['-C', workspace, 'track', element_name])
	1107	+ result.assert_success()
	1108	+
	1109	+
	1110	+@pytest.mark.datafiles(DATA_DIR)
	1111	+def test_external_open_other(cli, datafiles, tmpdir_factory):
	1112	+ # >From inside an external workspace, open another workspace
	1113	+ tmpdir1 = tmpdir_factory.mktemp('')
	1114	+ tmpdir2 = tmpdir_factory.mktemp('')
	1115	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1116	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1117	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1118	+
	1119	+ # Closing the other element first, because I'm too lazy to create an
	1120	+ # element without opening it
	1121	+ result = cli.run(project=project, args=['workspace', 'close', beta_element])
	1122	+ result.assert_success()
	1123	+
	1124	+ result = cli.run(project=project, args=[
	1125	+ '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element
	1126	+ ])
	1127	+ result.assert_success()
	1128	+
	1129	+
	1130	+@pytest.mark.datafiles(DATA_DIR)
	1131	+def test_external_close_other(cli, datafiles, tmpdir_factory):
	1132	+ # >From inside an external workspace, close the other workspace
	1133	+ tmpdir1 = tmpdir_factory.mktemp('')
	1134	+ tmpdir2 = tmpdir_factory.mktemp('')
	1135	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1136	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1137	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1138	+
	1139	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
	1140	+ result.assert_success()
	1141	+
	1142	+
	1143	+@pytest.mark.datafiles(DATA_DIR)
	1144	+def test_external_close_self(cli, datafiles, tmpdir_factory):
	1145	+ # >From inside an external workspace, close it
	1146	+ tmpdir1 = tmpdir_factory.mktemp('')
	1147	+ tmpdir2 = tmpdir_factory.mktemp('')
	1148	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1149	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1150	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1151	+
	1152	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', alpha_element])
	1153	+ result.assert_success()
	1154	+
	1155	+
	1156	+@pytest.mark.datafiles(DATA_DIR)
	1157	+def test_external_reset_other(cli, datafiles, tmpdir_factory):
	1158	+ tmpdir1 = tmpdir_factory.mktemp('')
	1159	+ tmpdir2 = tmpdir_factory.mktemp('')
	1160	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1161	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1162	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1163	+
	1164	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element])
	1165	+ result.assert_success()
	1166	+
	1167	+
	1168	+@pytest.mark.datafiles(DATA_DIR)
	1169	+def test_external_reset_self(cli, datafiles, tmpdir):
	1170	+ element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1171	+
	1172	+ # Command succeeds
	1173	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', element])
	1174	+ result.assert_success()
	1175	+
	1176	+ # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted)
	1177	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
	1178	+ result.assert_success()
	1179	+
	1180	+
	1181	+@pytest.mark.datafiles(DATA_DIR)
	1182	+def test_external_list(cli, datafiles, tmpdir_factory):
	1183	+ tmpdir = tmpdir_factory.mktemp('')
	1184	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1185	+ element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1186	+
	1187	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
	1188	+ result.assert_success()

tests/integration/shell.py

---

...	...	@@ -353,3 +353,29 @@ def test_integration_devices(cli, tmpdir, datafiles):
353	353
354	354	result = execute_shell(cli, project, ["true"], element=element_name)
355	355	assert result.exit_code == 0
	356	+
	357	+
	358	+# Test that a shell can be opened from an external workspace
	359	+@pytest.mark.datafiles(DATA_DIR)
	360	+@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
	361	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	362	+def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell):
	363	+ tmpdir = tmpdir_factory.mktemp("")
	364	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	365	+ element_name = 'autotools/amhello.bst'
	366	+ workspace_dir = os.path.join(str(tmpdir), 'workspace')
	367	+
	368	+ result = cli.run(project=project, args=[
	369	+ 'workspace', 'open', '--directory', workspace_dir, element_name
	370	+ ])
	371	+ result.assert_success()
	372	+
	373	+ result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
	374	+ result.assert_success()
	375	+
	376	+ command = ['shell']
	377	+ if build_shell == 'build':
	378	+ command.append('--build')
	379	+ command.extend([element_name, '--', 'true'])
	380	+ result = cli.run(project=project, cwd=workspace_dir, args=command)
	381	+ result.assert_success()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.