[Notes] [Git][BuildStream/buildstream][Qinusty/retries-should-fail] 9 commits: Report processing errors from tracking

[Qinusty <gitlab mg gitlab com>]

Title: GitLab

Qinusty pushed to branch Qinusty/retries-should-fail at BuildStream / buildstream

Commits:

• a38d9163
  by Valentin David at 2018-08-30T10:26:57Z

  Report processing errors from tracking
  
  Failures to write files when tracking were not reported.
  
  Fixes #533.
  

• 02138181
  by Tristan Van Berkom at 2018-08-30T11:36:00Z

  Merge branch 'valentindavid/post_tracking_errors' into 'master'
  
  Report processing errors from tracking
  
  Closes #533
  
  See merge request BuildStream/buildstream!747

• d23d4f3e
  by Tristan Van Berkom at 2018-08-30T12:02:27Z

  _artifactcache/artifactcache.py: Write the cache_size file atomically
  
  This is causing issues while the size file is being read and written
  simultaneously.
  
  The proper fix will be to read/add/save the file atomically and that
  will require locking, but this fix is a good stop gap to existing crashes.
  

• bb6aa5a5
  by Tristan Van Berkom at 2018-08-30T12:27:55Z

  Merge branch 'tristan/atomic-cache-size-file' into 'master'
  
  _artifactcache/artifactcache.py: Write the cache_size file atomically
  
  See merge request BuildStream/buildstream!762

• 680e4fe1
  by knownexus at 2018-08-30T13:37:55Z

  Replacing string 'bzr' with value from host tools
  

• 8cd719eb
  by Phillip Smyth at 2018-08-30T15:47:06Z

  Merge branch 'bzr_fix' into 'master'
  
  Replacing string 'bzr' with value from host tools
  
  See merge request BuildStream/buildstream!764

• 7e31aa8d
  by Josh Smith at 2018-08-30T16:07:27Z

  job.py: Modify retry messages to be FAIL
  
  This adjusts the message handler for the child processes to no longer
  override the message type.
  
  This also removes the ability for unhandled non BstError exceptions to
  retry.
  

• 3a1659ff
  by Josh Smith at 2018-08-30T16:07:27Z

  widget.py: Require message.logfile for log widget
  
  Prevent traceback in the case which a FAIL message is produced without a
  logfile being specified.
  
  Fixes: #619
  

• 6cbaeae7
  by Josh Smith at 2018-08-30T16:07:27Z

  tests: Modify tests to ensure retry FAILURE
  

9 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_frontend/widget.py
• buildstream/_scheduler/jobs/job.py
• buildstream/_scheduler/queues/trackqueue.py
• tests/frontend/track.py
• tests/sources/deb.py
• tests/sources/tar.py
• tests/sources/zip.py
• tests/testutils/repo/bzr.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -571,7 +571,7 @@ class ArtifactCache():
571	571	def _write_cache_size(self, size):
572	572	assert isinstance(size, int)
573	573	size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
574		- with open(size_file_path, "w") as f:
	574	+ with utils.save_file_atomic(size_file_path, "w") as f:
575	575	f.write(str(size))
576	576
577	577	# _read_cache_size()

buildstream/_frontend/widget.py

---

...	...	@@ -679,7 +679,7 @@ class LogLine(Widget):
679	679	text += '\n'
680	680	extra_nl = True
681	681
682		- if message.scheduler and message.message_type == MessageType.FAIL:
	682	+ if message.logfile and message.scheduler and message.message_type == MessageType.FAIL:
683	683	text += '\n'
684	684
685	685	if self.context is not None and not self.context.log_verbose:

buildstream/_scheduler/jobs/job.py

---

...	...	@@ -430,7 +430,8 @@ class Job():
430	430	self.message(MessageType.BUG, self.action_name,
431	431	elapsed=elapsed, detail=detail,
432	432	logfile=filename)
433		- self._child_shutdown(RC_FAIL)
	433	+ # Unhandled exceptions should permenantly fail
	434	+ self._child_shutdown(RC_PERM_FAIL)
434	435
435	436	else:
436	437	# No exception occurred in the action
...	...	@@ -509,11 +510,6 @@ class Job():
509	510	message.action_name = self.action_name
510	511	message.task_id = self._task_id
511	512
512		- if (message.message_type == MessageType.FAIL and
513		- self._tries <= self._max_retries and self._retry_flag):
514		- # Job will be retried, display failures as warnings in the frontend
515		- message.message_type = MessageType.WARN
516		-
517	513	# Send to frontend if appropriate
518	514	if context.silent_messages() and (message.message_type not in unconditional_messages):
519	515	return

buildstream/_scheduler/queues/trackqueue.py

---

...	...	@@ -58,18 +58,9 @@ class TrackQueue(Queue):
58	58	# Set the new refs in the main process one by one as they complete
59	59	for unique_id, new_ref in result:
60	60	source = _plugin_lookup(unique_id)
61		- try:
62		- # We appear processed if at least one source has changed
63		- if source._save_ref(new_ref):
64		- changed = True
65		- except SourceError as e:
66		- # FIXME: We currently dont have a clear path to
67		- # fail the scheduler from the main process, so
68		- # this will just warn and BuildStream will exit
69		- # with a success code.
70		- #
71		- source.warn("Failed to update project file",
72		- detail="{}".format(e))
	61	+ # We appear processed if at least one source has changed
	62	+ if source._save_ref(new_ref):
	63	+ changed = True
73	64
74	65	element._tracking_done()
75	66

tests/frontend/track.py

---

	1	+import stat
1	2	import os
2	3	import pytest
3	4	from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
...	...	@@ -634,3 +635,36 @@ def test_track_junction_included(cli, tmpdir, datafiles, ref_storage, kind):
634	635
635	636	result = cli.run(project=project, args=['track', 'junction.bst'])
636	637	result.assert_success()
	638	+
	639	+
	640	+@pytest.mark.datafiles(DATA_DIR)
	641	+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
	642	+def test_track_error_cannot_write_file(cli, tmpdir, datafiles, kind):
	643	+ if os.geteuid() == 0:
	644	+ pytest.skip("This is not testable with root permissions")
	645	+
	646	+ project = str(datafiles)
	647	+ dev_files_path = os.path.join(project, 'files', 'dev-files')
	648	+ element_path = os.path.join(project, 'elements')
	649	+ element_name = 'track-test-{}.bst'.format(kind)
	650	+
	651	+ configure_project(project, {
	652	+ 'ref-storage': 'inline'
	653	+ })
	654	+
	655	+ repo = create_repo(kind, str(tmpdir))
	656	+ ref = repo.create(dev_files_path)
	657	+
	658	+ element_full_path = os.path.join(element_path, element_name)
	659	+ generate_element(repo, element_full_path)
	660	+
	661	+ st = os.stat(element_path)
	662	+ try:
	663	+ read_mask = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH
	664	+ os.chmod(element_path, stat.S_IMODE(st.st_mode) & ~read_mask)
	665	+
	666	+ result = cli.run(project=project, args=['track', element_name])
	667	+ result.assert_main_error(ErrorDomain.STREAM, None)
	668	+ result.assert_task_error(ErrorDomain.SOURCE, 'save-ref-error')
	669	+ finally:
	670	+ os.chmod(element_path, stat.S_IMODE(st.st_mode))

tests/sources/deb.py

---

...	...	@@ -56,7 +56,7 @@ def test_fetch_bad_url(cli, tmpdir, datafiles):
56	56	result = cli.run(project=project, args=[
57	57	'fetch', 'target.bst'
58	58	])
59		- assert "Try #" in result.stderr
	59	+ assert "FAILURE Try #" in result.stderr
60	60	result.assert_main_error(ErrorDomain.STREAM, None)
61	61	result.assert_task_error(ErrorDomain.SOURCE, None)
62	62

tests/sources/tar.py

---

...	...	@@ -67,7 +67,7 @@ def test_fetch_bad_url(cli, tmpdir, datafiles):
67	67	result = cli.run(project=project, args=[
68	68	'fetch', 'target.bst'
69	69	])
70		- assert "Try #" in result.stderr
	70	+ assert "FAILURE Try #" in result.stderr
71	71	result.assert_main_error(ErrorDomain.STREAM, None)
72	72	result.assert_task_error(ErrorDomain.SOURCE, None)
73	73

tests/sources/zip.py

---

...	...	@@ -53,7 +53,7 @@ def test_fetch_bad_url(cli, tmpdir, datafiles):
53	53	result = cli.run(project=project, args=[
54	54	'fetch', 'target.bst'
55	55	])
56		- assert "Try #" in result.stderr
	56	+ assert "FAILURE Try #" in result.stderr
57	57	result.assert_main_error(ErrorDomain.STREAM, None)
58	58	result.assert_task_error(ErrorDomain.SOURCE, None)
59	59

tests/testutils/repo/bzr.py

---

...	...	@@ -2,6 +2,7 @@ import os
2	2	import subprocess
3	3	import pytest
4	4
	5	+from buildstream import utils
5	6	from .repo import Repo
6	7	from ..site import HAVE_BZR
7	8
...	...	@@ -16,15 +17,16 @@ class Bzr(Repo):
16	17	if not HAVE_BZR:
17	18	pytest.skip("bzr is not available")
18	19	super(Bzr, self).__init__(directory, subdir)
	20	+ self.bzr = utils.get_host_tool('bzr')
19	21
20	22	def create(self, directory):
21	23	branch_dir = os.path.join(self.repo, 'trunk')
22	24
23		- subprocess.call(['bzr', 'init-repo', self.repo], env=BZR_ENV)
24		- subprocess.call(['bzr', 'init', branch_dir], env=BZR_ENV)
	25	+ subprocess.call([self.bzr, 'init-repo', self.repo], env=BZR_ENV)
	26	+ subprocess.call([self.bzr, 'init', branch_dir], env=BZR_ENV)
25	27	self.copy_directory(directory, branch_dir)
26		- subprocess.call(['bzr', 'add', '.'], env=BZR_ENV, cwd=branch_dir)
27		- subprocess.call(['bzr', 'commit', '--message="Initial commit"'],
	28	+ subprocess.call([self.bzr, 'add', '.'], env=BZR_ENV, cwd=branch_dir)
	29	+ subprocess.call([self.bzr, 'commit', '--message="Initial commit"'],
28	30	env=BZR_ENV, cwd=branch_dir)
29	31
30	32	return self.latest_commit()
...	...	@@ -42,7 +44,7 @@ class Bzr(Repo):
42	44
43	45	def latest_commit(self):
44	46	output = subprocess.check_output([
45		- 'bzr', 'version-info',
	47	+ self.bzr, 'version-info',
46	48	'--custom', '--template={revno}',
47	49	os.path.join(self.repo, 'trunk')
48	50	], env=BZR_ENV)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.