[Notes] [Git][BuildStream/buildstream][relative_workspaces] Implementing relative workspaces

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch relative_workspaces at BuildStream / buildstream

Commits:

• 8d5f2338
  by Phillip Smyth at 2018-07-27T14:39:26Z

  Implementing relative workspaces
  

6 changed files:

• buildstream/_frontend/widget.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/element.py
• tests/frontend/workspace.py
• tests/integration/shell.py

Changes:

buildstream/_frontend/widget.py

---

...	...	@@ -415,7 +415,7 @@ class LogLine(Widget):
415	415	if "%{workspace-dirs" in format_:
416	416	workspace = element._get_workspace()
417	417	if workspace is not None:
418		- path = workspace.path.replace(os.getenv('HOME', '/root'), '~')
	418	+ path = workspace.get_absolute_path().replace(os.getenv('HOME', '/root'), '~')
419	419	line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path))
420	420	else:
421	421	line = p.fmt_subst(

buildstream/_stream.py

---

...	...	@@ -442,7 +442,7 @@ class Stream():
442	442	workspace = workspaces.get_workspace(target._get_full_name())
443	443	if workspace:
444	444	raise StreamError("Workspace '{}' is already defined at: {}"
445		- .format(target.name, workspace.path))
	445	+ .format(target.name, workspace.get_absolute_path()))
446	446
447	447	# If we're going to checkout, we need at least a fetch,
448	448	# if we were asked to track first, we're going to fetch anyway.
...	...	@@ -460,14 +460,14 @@ class Stream():
460	460	"source.")
461	461
462	462	try:
463		- os.makedirs(directory, exist_ok=True)
	463	+ os.makedirs(workdir, exist_ok=True)
464	464	except OSError as e:
465	465	raise StreamError("Failed to create workspace directory: {}".format(e)) from e
466	466
467	467	workspaces.create_workspace(target._get_full_name(), workdir)
468	468
469	469	if not no_checkout:
470		- with target.timed_activity("Staging sources to {}".format(directory)):
	470	+ with target.timed_activity("Staging sources to {}".format(workdir)):
471	471	target._open_workspace()
472	472
473	473	workspaces.save_config()
...	...	@@ -488,12 +488,12 @@ class Stream():
488	488	# Remove workspace directory if prompted
489	489	if remove_dir:
490	490	with self._context.timed_activity("Removing workspace directory {}"
491		- .format(workspace.path)):
	491	+ .format(workspace.get_absolute_path())):
492	492	try:
493		- shutil.rmtree(workspace.path)
	493	+ shutil.rmtree(workspace.get_absolute_path())
494	494	except OSError as e:
495	495	raise StreamError("Could not remove '{}': {}"
496		- .format(workspace.path, e)) from e
	496	+ .format(workspace.get_absolute_path(), e)) from e
497	497
498	498	# Delete the workspace and save the configuration
499	499	workspaces.delete_workspace(element_name)
...	...	@@ -536,28 +536,30 @@ class Stream():
536	536
537	537	for element in elements:
538	538	workspace = workspaces.get_workspace(element._get_full_name())
539		-
	539	+ workspace_path = workspace.get_absolute_path()
540	540	if soft:
541	541	workspace.prepared = False
542	542	self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
543		- .format(element.name, workspace.path))
	543	+ .format(element.name, workspace_path))
544	544	continue
545	545
546	546	with element.timed_activity("Removing workspace directory {}"
547		- .format(workspace.path)):
	547	+ .format(workspace_path)):
548	548	try:
549		- shutil.rmtree(workspace.path)
	549	+ shutil.rmtree(workspace_path)
550	550	except OSError as e:
551	551	raise StreamError("Could not remove '{}': {}"
552		- .format(workspace.path, e)) from e
	552	+ .format(workspace_path, e)) from e
553	553
554	554	workspaces.delete_workspace(element._get_full_name())
555		- workspaces.create_workspace(element._get_full_name(), workspace.path)
	555	+ workspaces.create_workspace(element._get_full_name(), workspace_path)
556	556
557		- with element.timed_activity("Staging sources to {}".format(workspace.path)):
	557	+ with element.timed_activity("Staging sources to {}".format(workspace_path)):
558	558	element._open_workspace()
559	559
560		- self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
	560	+ self._message(MessageType.INFO,
	561	+ "Reset workspace for {} at: {}".format(element.name,
	562	+ workspace_path))
561	563
562	564	workspaces.save_config()
563	565
...	...	@@ -594,7 +596,7 @@ class Stream():
594	596	for element_name, workspace_ in self._context.get_workspaces().list():
595	597	workspace_detail = {
596	598	'element': element_name,
597		- 'directory': workspace_.path,
	599	+ 'directory': workspace_.get_absolute_path(),
598	600	}
599	601	workspaces.append(workspace_detail)
600	602

buildstream/_workspaces.py

---

...	...	@@ -56,7 +56,7 @@ class Workspace():
56	56	def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
57	57	self.prepared = prepared
58	58	self.last_successful = last_successful
59		- self.path = path
	59	+ self._path = path
60	60	self.running_files = running_files if running_files is not None else {}
61	61
62	62	self._toplevel_project = toplevel_project
...	...	@@ -70,8 +70,9 @@ class Workspace():
70	70	# (dict) A dict representation of the workspace
71	71	#
72	72	def to_dict(self):
73		- return {key: val for key, val in self.__dict__.items()
74		- if key in _WORKSPACE_MEMBERS and val is not None}
	73	+ # Strip the leading underscore from _path
	74	+ return {key.lstrip("_"): val for key, val in self.__dict__.items()
	75	+ if key.lstrip("_") in _WORKSPACE_MEMBERS and val is not None}
75	76
76	77	# from_dict():
77	78	#
...	...	@@ -133,7 +134,7 @@ class Workspace():
133	134	if os.path.isdir(fullpath):
134	135	utils.copy_files(fullpath, directory)
135	136	else:
136		- destfile = os.path.join(directory, os.path.basename(self.path))
	137	+ destfile = os.path.join(directory, os.path.basename(self.get_absolute_path()))
137	138	utils.safe_copy(fullpath, destfile)
138	139
139	140	# add_running_files()
...	...	@@ -189,7 +190,7 @@ class Workspace():
189	190	filelist = utils.list_relative_paths(fullpath)
190	191	filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
191	192	else:
192		- filelist = [(self.path, fullpath)]
	193	+ filelist = [(self.get_absolute_path(), fullpath)]
193	194
194	195	self._key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
195	196
...	...	@@ -200,7 +201,7 @@ class Workspace():
200	201	# Returns: The absolute path of the element's workspace.
201	202	#
202	203	def get_absolute_path(self):
203		- return os.path.join(self._toplevel_project.directory, self.path)
	204	+ return os.path.join(self._toplevel_project.directory, self._path)
204	205
205	206
206	207	# Workspaces()
...	...	@@ -236,6 +237,9 @@ class Workspaces():
236	237	# path (str) - The path in which the workspace should be kept
237	238	#
238	239	def create_workspace(self, element_name, path):
	240	+ if path.startswith(self._toplevel_project.directory):
	241	+ path = os.path.relpath(path, self._toplevel_project.directory)
	242	+
239	243	self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
240	244
241	245	return self._workspaces[element_name]

buildstream/element.py

---

...	...	@@ -1309,7 +1309,7 @@ class Element(Plugin):
1309	1309	# If mount_workspaces is set and we're doing incremental builds,
1310	1310	# the workspace is already mounted into the sandbox.
1311	1311	if not (mount_workspaces and self.__can_build_incrementally()):
1312		- with self.timed_activity("Staging local files at {}".format(workspace.path)):
	1312	+ with self.timed_activity("Staging local files at {}".format(workspace.get_absolute_path())):
1313	1313	workspace.stage(directory)
1314	1314	else:
1315	1315	# No workspace, stage directly
...	...	@@ -1459,7 +1459,7 @@ class Element(Plugin):
1459	1459	sandbox_path = os.path.join(sandbox_root,
1460	1460	self.__staged_sources_directory.lstrip(os.sep))
1461	1461	try:
1462		- utils.copy_files(workspace.path, sandbox_path)
	1462	+ utils.copy_files(workspace.get_absolute_path(), sandbox_path)
1463	1463	except UtilError as e:
1464	1464	self.warn("Failed to preserve workspace state for failed build sysroot: {}"
1465	1465	.format(e))
...	...	@@ -1752,7 +1752,7 @@ class Element(Plugin):
1752	1752	source._init_workspace(temp)
1753	1753
1754	1754	# Now hardlink the files into the workspace target.
1755		- utils.link_files(temp, workspace.path)
	1755	+ utils.link_files(temp, workspace.get_absolute_path())
1756	1756
1757	1757	# _get_workspace():
1758	1758	#

tests/frontend/workspace.py

---

...	...	@@ -18,7 +18,10 @@ DATA_DIR = os.path.join(
18	18	)
19	19
20	20
21		-def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
	21	+def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', rerun=False, new_dir=None):
	22	+ if os.path.isdir(os.path.join(str(tmpdir), 'repo')):
	23	+ shutil.rmtree(os.path.join(str(tmpdir), 'repo'))
	24	+
22	25	project = os.path.join(datafiles.dirname, datafiles.basename)
23	26	bin_files_path = os.path.join(project, 'files', 'bin-files')
24	27	element_path = os.path.join(project, 'elements')
...	...	@@ -45,11 +48,12 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
45	48	element_name))
46	49
47	50	# Assert that there is no reference, a track & fetch is needed
48		- state = cli.get_element_state(project, element_name)
49		- if track:
50		- assert state == 'no reference'
51		- else:
52		- assert state == 'fetch needed'
	51	+ if not rerun:
	52	+ state = cli.get_element_state(project, element_name)
	53	+ if track:
	54	+ assert state == 'no reference'
	55	+ else:
	56	+ assert state == 'fetch needed'
53	57
54	58	# Now open the workspace, this should have the effect of automatically
55	59	# tracking & fetching the source from the repo.
...	...	@@ -59,15 +63,19 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
59	63	args.extend([element_name, workspace])
60	64
61	65	result = cli.run(project=project, args=args)
62		- result.assert_success()
63	66
64		- # Assert that we are now buildable because the source is
65		- # now cached.
66		- assert cli.get_element_state(project, element_name) == 'buildable'
	67	+ if not rerun:
	68	+ result.assert_success()
67	69
68		- # Check that the executable hello file is found in the workspace
69		- filename = os.path.join(workspace, 'usr', 'bin', 'hello')
70		- assert os.path.exists(filename)
	70	+ # Assert that we are now buildable because the source is
	71	+ # now cached.
	72	+ assert cli.get_element_state(project, element_name) == 'buildable'
	73	+
	74	+ # Check that the executable hello file is found in the workspace
	75	+ filename = os.path.join(workspace, 'usr', 'bin', 'hello')
	76	+ assert os.path.exists(filename)
	77	+ else:
	78	+ result.assert_main_error(ErrorDomain.STREAM, None)
71	79
72	80	return (element_name, project, workspace)
73	81
...	...	@@ -78,6 +86,43 @@ def test_open(cli, tmpdir, datafiles, kind):
78	86	open_workspace(cli, tmpdir, datafiles, kind, False)
79	87
80	88
	89	+@pytest.mark.datafiles(DATA_DIR)
	90	+@pytest.mark.parametrize("kind", repo_kinds)
	91	+def test_open_after_move_interally(cli, tmpdir, datafiles, kind):
	92	+ open_workspace(cli, tmpdir, datafiles, kind, False)
	93	+ workspace = os.path.join(str(tmpdir), 'workspace')
	94	+ shutil.move(workspace, os.path.join(str(tmpdir), 'workspace2'))
	95	+ open_workspace(cli, tmpdir, datafiles, kind, False, '', True)
	96	+ filename = os.path.join(workspace + '2', 'usr', 'bin', 'hello')
	97	+ assert os.path.exists(filename)
	98	+
	99	+
	100	+@pytest.mark.datafiles(DATA_DIR)
	101	+@pytest.mark.parametrize("kind", repo_kinds)
	102	+def test_open_after_move_externally(cli, tmpdir, datafiles, kind):
	103	+ open_workspace(cli, tmpdir, datafiles, kind, False)
	104	+ workspace = os.path.join(str(tmpdir), 'workspace')
	105	+ tmp_dir = os.path.join(str(tmpdir), '2')
	106	+ shutil.move(workspace, os.path.join(tmp_dir, 'workspace'))
	107	+ open_workspace(cli, tmpdir, datafiles, kind, False, '', True, tmp_dir)
	108	+ filename = os.path.join(os.path.join(tmp_dir, 'workspace'), 'usr', 'bin', 'hello')
	109	+ assert os.path.exists(filename)
	110	+ shutil.rmtree(tmp_dir)
	111	+
	112	+
	113	+@pytest.mark.datafiles(DATA_DIR)
	114	+@pytest.mark.parametrize("kind", repo_kinds)
	115	+def test_open_after_move_project(cli, tmpdir, datafiles, kind):
	116	+ open_workspace(cli, tmpdir, datafiles, kind, False)
	117	+ workspace = os.path.join(str(tmpdir), 'workspace')
	118	+ tmp_dir = os.path.join(str(tmpdir), '..', 'new_dir')
	119	+ shutil.move(workspace, os.path.join(tmp_dir, 'workspace'))
	120	+ open_workspace(cli, tmpdir, datafiles, kind, False, '', True, tmp_dir)
	121	+ filename = os.path.join(os.path.join(tmp_dir, 'workspace'), 'usr', 'bin', 'hello')
	122	+ assert os.path.exists(filename)
	123	+ shutil.rmtree(tmp_dir)
	124	+
	125	+
81	126	@pytest.mark.datafiles(DATA_DIR)
82	127	def test_open_bzr_customize(cli, tmpdir, datafiles):
83	128	element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "bzr", False)

tests/integration/shell.py

---

...	...	@@ -330,6 +330,7 @@ def test_sysroot_workspace_visible(cli, tmpdir, datafiles):
330	330	# Obtain a copy of the hello.c content from the workspace
331	331	#
332	332	workspace_hello_path = os.path.join(cli.directory, 'workspace', 'hello.c')
	333	+
333	334	assert os.path.exists(workspace_hello_path)
334	335	with open(workspace_hello_path, 'r') as f:
335	336	workspace_hello = f.read()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.