[Notes] [Git][BuildStream/buildstream][tristan/backport-logline-dups-1.2] 3 commits: git.py: Handle concurrent download completions properly

[Tristan Van Berkom <gitlab mg gitlab com>]

Title: GitLab

Tristan Van Berkom pushed to branch tristan/backport-logline-dups-1.2 at BuildStream / buildstream

Commits:

• 95cc9793
  by Tristan Van Berkom at 2018-07-29T17:21:24Z

  git.py: Handle concurrent download completions properly
  
  Use os.rename() to rename the cloned temporary repository into
  place in the source cache, and issue a STATUS message when discarding
  a duplicate clone, in the case where the same repository is cloned
  twice in parallel.
  
  The problem with using shutil.move() is that it will create the source
  directory in a subdirectory of the destination when the destination
  exists, so it's behavior depends on whether the destination exists.
  
  This shutil.move() behavior has so far hidden the race condition
  where a duplicate repo is created in a subdirectory, as you need
  to have three concurrent downloads of the same repo in order to
  trigger the error.
  
  This fixes issue #503
  

• 55aacefa
  by Tristan Van Berkom at 2018-07-29T19:53:57Z

  Merge branch 'tristan/fix-spurious-errors-1.2' into 'bst-1.2'
  
  git.py: Handle concurrent download completions properly
  
  See merge request BuildStream/buildstream!584

• 12fea82e
  by Tristan Maat at 2018-07-29T19:54:50Z

  buildelement.py: Only display one activity message (#507)
  

2 changed files:

• buildstream/buildelement.py
• buildstream/plugins/sources/git.py

Changes:

buildstream/buildelement.py

---

1	1	#
2	2	# Copyright (C) 2016 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
3	4	#
4	5	# This program is free software; you can redistribute it and/or
5	6	# modify it under the terms of the GNU Lesser General Public
...	...	@@ -204,8 +205,9 @@ class BuildElement(Element):
204	205	def prepare(self, sandbox):
205	206	commands = self.__commands['configure-commands']
206	207	if commands:
207		- for cmd in commands:
208		- self.__run_command(sandbox, cmd, 'configure-commands')
	208	+ with self.timed_activity("Running configure-commands"):
	209	+ for cmd in commands:
	210	+ self.__run_command(sandbox, cmd, 'configure-commands')
209	211
210	212	def generate_script(self):
211	213	script = ""
...	...	@@ -231,13 +233,12 @@ class BuildElement(Element):
231	233	return commands
232	234
233	235	def __run_command(self, sandbox, cmd, cmd_name):
234		- with self.timed_activity("Running {}".format(cmd_name)):
235		- self.status("Running {}".format(cmd_name), detail=cmd)
236		-
237		- # Note the -e switch to 'sh' means to exit with an error
238		- # if any untested command fails.
239		- #
240		- exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
241		- SandboxFlags.ROOT_READ_ONLY)
242		- if exitcode != 0:
243		- raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode))
	236	+ self.status("Running {}".format(cmd_name), detail=cmd)
	237	+
	238	+ # Note the -e switch to 'sh' means to exit with an error
	239	+ # if any untested command fails.
	240	+ #
	241	+ exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
	242	+ SandboxFlags.ROOT_READ_ONLY)
	243	+ if exitcode != 0:
	244	+ raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode))

buildstream/plugins/sources/git.py

---

...	...	@@ -71,8 +71,8 @@ git - stage files from a git repository
71	71	"""
72	72
73	73	import os
	74	+import errno
74	75	import re
75		-import shutil
76	76	from collections import Mapping
77	77	from io import StringIO
78	78
...	...	@@ -119,11 +119,21 @@ class GitMirror(SourceFetcher):
119	119	fail="Failed to clone git repository {}".format(url),
120	120	fail_temporarily=True)
121	121
	122	+ # Attempt atomic rename into destination, this will fail if
	123	+ # another process beat us to the punch
122	124	try:
123		- shutil.move(tmpdir, self.mirror)
124		- except (shutil.Error, OSError) as e:
125		- raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}'"
126		- .format(self.source, url, tmpdir, self.mirror)) from e
	125	+ os.rename(tmpdir, self.mirror)
	126	+ except OSError as e:
	127	+
	128	+ # When renaming and the destination repo already exists, os.rename()
	129	+ # will fail with ENOTEMPTY, since an empty directory will be silently
	130	+ # replaced
	131	+ if e.errno == errno.ENOTEMPTY:
	132	+ self.source.status("{}: Discarding duplicate clone of {}"
	133	+ .format(self.source, url))
	134	+ else:
	135	+ raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}': {}"
	136	+ .format(self.source, url, tmpdir, self.mirror, e)) from e
127	137
128	138	def _fetch(self, alias_override=None):
129	139	url = self.source.translate_url(self.url, alias_override=alias_override)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.