[Notes] [Git][BuildStream/buildstream][tristan/cleanup-staging-tempdir] element.py: Cleanup temporary staging directories on termination

[Tristan Van Berkom <gitlab mg gitlab com>]

Title: GitLab

Tristan Van Berkom pushed to branch tristan/cleanup-staging-tempdir at BuildStream / buildstream

Commits:

• 19adc9e5
  by Tristan Van Berkom at 2019-01-16T20:48:10Z

  element.py: Cleanup temporary staging directories on termination
  
  Use utils._tempdir() which in turn uses _signals.terminator() for
  this purpose.
  
  This also changes utils._tempdir() to use utils._force_rmtree()
  so that it is safe to use for cleaning up the staging directory.
  
  This patch fixes orphaned temporary directories being left behind
  in the artifact cache directory at forceful termination time.
  

2 changed files:

• buildstream/element.py
• buildstream/utils.py

Changes:

buildstream/element.py

---

...	...	@@ -1398,43 +1398,40 @@ class Element(Plugin):
1398	1398	# usebuildtree (bool): use a the elements build tree as its source.
1399	1399	#
1400	1400	def _stage_sources_at(self, vdirectory, mount_workspaces=True, usebuildtree=False):
1401		- with self.timed_activity("Staging sources", silent_nested=True):
	1401	+ context = self._get_context()
	1402	+
	1403	+ # It's advantageous to have this temporary directory on
	1404	+ # the same file system as the rest of our cache.
	1405	+ with self.timed_activity("Staging sources", silent_nested=True), \
	1406	+ utils._tempdir(dir=context.artifactdir, prefix='staging-temp') as temp_staging_directory:
	1407	+
	1408	+ import_dir = temp_staging_directory
	1409	+
1402	1410	if not isinstance(vdirectory, Directory):
1403	1411	vdirectory = FileBasedDirectory(vdirectory)
1404	1412	if not vdirectory.is_empty():
1405	1413	raise ElementError("Staging directory '{}' is not empty".format(vdirectory))
1406	1414
1407		- # It's advantageous to have this temporary directory on
1408		- # the same filing system as the rest of our cache.
1409		- temp_staging_location = os.path.join(self._get_context().artifactdir, "staging_temp")
1410		- temp_staging_directory = tempfile.mkdtemp(prefix=temp_staging_location)
1411		- import_dir = temp_staging_directory
1412		-
1413		- try:
1414		- workspace = self._get_workspace()
1415		- if workspace:
1416		- # If mount_workspaces is set and we're doing incremental builds,
1417		- # the workspace is already mounted into the sandbox.
1418		- if not (mount_workspaces and self.__can_build_incrementally()):
1419		- with self.timed_activity("Staging local files at {}"
1420		- .format(workspace.get_absolute_path())):
1421		- workspace.stage(temp_staging_directory)
1422		- # Check if we have a cached buildtree to use
1423		- elif usebuildtree:
1424		- artifact_base, _ = self.__extract()
1425		- import_dir = os.path.join(artifact_base, 'buildtree')
1426		- else:
1427		- # No workspace or cached buildtree, stage source directly
1428		- for source in self.sources():
1429		- source._stage(temp_staging_directory)
	1415	+ workspace = self._get_workspace()
	1416	+ if workspace:
	1417	+ # If mount_workspaces is set and we're doing incremental builds,
	1418	+ # the workspace is already mounted into the sandbox.
	1419	+ if not (mount_workspaces and self.__can_build_incrementally()):
	1420	+ with self.timed_activity("Staging local files at {}"
	1421	+ .format(workspace.get_absolute_path())):
	1422	+ workspace.stage(temp_staging_directory)
	1423	+
	1424	+ # Check if we have a cached buildtree to use
	1425	+ elif usebuildtree:
	1426	+ artifact_base, _ = self.__extract()
	1427	+ import_dir = os.path.join(artifact_base, 'buildtree')
	1428	+ else:
	1429	+ # No workspace or cached buildtree, stage source directly
	1430	+ for source in self.sources():
	1431	+ source._stage(temp_staging_directory)
1430	1432
1431		- vdirectory.import_files(import_dir)
	1433	+ vdirectory.import_files(import_dir)
1432	1434
1433		- finally:
1434		- # Staging may produce directories with less than 'rwx' permissions
1435		- # for the owner, which breaks tempfile. _force_rmtree will deal
1436		- # with these.
1437		- utils._force_rmtree(temp_staging_directory)
1438	1435	# Ensure deterministic mtime of sources at build time
1439	1436	vdirectory.set_deterministic_mtime()
1440	1437	# Ensure deterministic owners of sources at build time

buildstream/utils.py

---

...	...	@@ -1023,7 +1023,7 @@ def _tempdir(suffix="", prefix="tmp", dir=None): # pylint: disable=redefined-bu
1023	1023
1024	1024	def cleanup_tempdir():
1025	1025	if os.path.isdir(tempdir):
1026		- shutil.rmtree(tempdir)
	1026	+ _force_rmtree(tempdir)
1027	1027
1028	1028	try:
1029	1029	with _signals.terminator(cleanup_tempdir):

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.