[Notes] [Git][BuildStream/buildstream][valentindavid/script-artifact-corruption-1.2] Fix cache corruption by scripts when layout and integration commands are used

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/script-artifact-corruption-1.2 at BuildStream / buildstream

Commits:

• 97376136
  by Valentin David at 2018-11-08T12:49:45Z

  Fix cache corruption by scripts when layout and integration commands are used
  
  Root directory was marked as a non-artifact mount, so not using
  SafeHardLink.  However integration commands executed with write access
  to the root directory.
  
  Fixes #749
  

10 changed files:

• buildstream/scriptelement.py
• + tests/integration/project/elements/script/corruption-2.bst
• + tests/integration/project/elements/script/corruption-image.bst
• + tests/integration/project/elements/script/corruption-integration.bst
• + tests/integration/project/elements/script/corruption.bst
• + tests/integration/project/elements/script/marked-tmpdir.bst
• + tests/integration/project/elements/script/no-tmpdir.bst
• + tests/integration/project/elements/script/tmpdir.bst
• + tests/integration/project/files/canary
• tests/integration/script.py

Changes:

buildstream/scriptelement.py

---

...	...	@@ -201,16 +201,20 @@ class ScriptElement(Element):
201	201	# Setup environment
202	202	sandbox.set_environment(self.get_environment())
203	203
	204	+ # Tell the sandbox to mount the install root
	205	+ directories = {self.__install_root: False}
	206	+
204	207	# Mark the artifact directories in the layout
205	208	for item in self.__layout:
206		- if item['destination'] != '/':
207		- if item['element']:
208		- sandbox.mark_directory(item['destination'], artifact=True)
209		- else:
210		- sandbox.mark_directory(item['destination'])
211		-
212		- # Tell the sandbox to mount the install root
213		- sandbox.mark_directory(self.__install_root)
	209	+ destination = item['destination']
	210	+ was_artifact = directories.get(destination, False)
	211	+ directories[destination] = item['element'] or was_artifact
	212	+
	213	+ for directory, artifact in directories.items():
	214	+ # Root does not need to be marked as it is always mounted
	215	+ # with artifact (unless explicitly marked non-artifact)
	216	+ if directory != '/':
	217	+ sandbox.mark_directory(directory, artifact=artifact)
214	218
215	219	def stage(self, sandbox):
216	220

tests/integration/project/elements/script/corruption-2.bst

---

	1	+kind: script
	2	+
	3	+depends:
	4	+- filename: base.bst
	5	+ type: build
	6	+- filename: script/corruption-image.bst
	7	+ type: build
	8	+
	9	+config:
	10	+ commands:
	11	+ - echo smashed >>/canary

tests/integration/project/elements/script/corruption-image.bst

---

	1	+kind: import
	2	+sources:
	3	+- kind: local
	4	+ path: files/canary

tests/integration/project/elements/script/corruption-integration.bst

---

	1	+kind: stack
	2	+
	3	+public:
	4	+ bst:
	5	+ integration-commands:
	6	+ - echo smashed >>/canary
	7	+

tests/integration/project/elements/script/corruption.bst

---

	1	+kind: script
	2	+
	3	+depends:
	4	+- filename: base.bst
	5	+ type: build
	6	+- filename: script/corruption-image.bst
	7	+ type: build
	8	+- filename: script/corruption-integration.bst
	9	+ type: build
	10	+
	11	+variables:
	12	+ install-root: "/"
	13	+
	14	+config:
	15	+ layout:
	16	+ - element: base.bst
	17	+ destination: "/"
	18	+ - element: script/corruption-image.bst
	19	+ destination: "/"
	20	+ - element: script/corruption-integration.bst
	21	+ destination: "/"

tests/integration/project/elements/script/marked-tmpdir.bst

---

	1	+kind: compose
	2	+
	3	+depends:
	4	+- filename: base.bst
	5	+ type: build
	6	+
	7	+public:
	8	+ bst:
	9	+ split-rules:
	10	+ remove:
	11	+ - "/tmp/**"
	12	+ - "/tmp"

tests/integration/project/elements/script/no-tmpdir.bst

---

	1	+kind: filter
	2	+
	3	+depends:
	4	+- filename: script/marked-tmpdir.bst
	5	+ type: build
	6	+
	7	+config:
	8	+ exclude:
	9	+ - remove
	10	+ include-orphans: True
	11	+
	12	+

tests/integration/project/elements/script/tmpdir.bst

---

	1	+kind: script
	2	+
	3	+depends:
	4	+- filename: script/no-tmpdir.bst
	5	+ type: build
	6	+
	7	+config:
	8	+ commands:
	9	+ - |
	10	+ mkdir -p /tmp/blah

tests/integration/project/files/canary

---

1

+alive

tests/integration/script.py

---

...	...	@@ -155,3 +155,70 @@ def test_script_layout(cli, tmpdir, datafiles):
155	155	text = f.read()
156	156
157	157	assert text == "Hi\n"
	158	+
	159	+
	160	+@pytest.mark.datafiles(DATA_DIR)
	161	+def test_regression_cache_corruption(cli, tmpdir, datafiles):
	162	+ project = str(datafiles)
	163	+ checkout_original = os.path.join(cli.directory, 'checkout-original')
	164	+ checkout_after = os.path.join(cli.directory, 'checkout-after')
	165	+ element_name = 'script/corruption.bst'
	166	+ canary_element_name = 'script/corruption-image.bst'
	167	+
	168	+ res = cli.run(project=project, args=['build', canary_element_name])
	169	+ assert res.exit_code == 0
	170	+
	171	+ res = cli.run(project=project, args=['checkout', canary_element_name,
	172	+ checkout_original])
	173	+ assert res.exit_code == 0
	174	+
	175	+ with open(os.path.join(checkout_original, 'canary')) as f:
	176	+ assert f.read() == 'alive\n'
	177	+
	178	+ res = cli.run(project=project, args=['build', element_name])
	179	+ assert res.exit_code == 0
	180	+
	181	+ res = cli.run(project=project, args=['checkout', canary_element_name,
	182	+ checkout_after])
	183	+ assert res.exit_code == 0
	184	+
	185	+ with open(os.path.join(checkout_after, 'canary')) as f:
	186	+ assert f.read() == 'alive\n'
	187	+
	188	+
	189	+@pytest.mark.datafiles(DATA_DIR)
	190	+def test_regression_tmpdir(cli, tmpdir, datafiles):
	191	+ project = str(datafiles)
	192	+ element_name = 'script/tmpdir.bst'
	193	+
	194	+ res = cli.run(project=project, args=['build', element_name])
	195	+ assert res.exit_code == 0
	196	+
	197	+
	198	+@pytest.mark.datafiles(DATA_DIR)
	199	+def test_regression_cache_corruption_2(cli, tmpdir, datafiles):
	200	+ project = str(datafiles)
	201	+ checkout_original = os.path.join(cli.directory, 'checkout-original')
	202	+ checkout_after = os.path.join(cli.directory, 'checkout-after')
	203	+ element_name = 'script/corruption-2.bst'
	204	+ canary_element_name = 'script/corruption-image.bst'
	205	+
	206	+ res = cli.run(project=project, args=['build', canary_element_name])
	207	+ assert res.exit_code == 0
	208	+
	209	+ res = cli.run(project=project, args=['checkout', canary_element_name,
	210	+ checkout_original])
	211	+ assert res.exit_code == 0
	212	+
	213	+ with open(os.path.join(checkout_original, 'canary')) as f:
	214	+ assert f.read() == 'alive\n'
	215	+
	216	+ res = cli.run(project=project, args=['build', element_name])
	217	+ assert res.exit_code == 0
	218	+
	219	+ res = cli.run(project=project, args=['checkout', canary_element_name,
	220	+ checkout_after])
	221	+ assert res.exit_code == 0
	222	+
	223	+ with open(os.path.join(checkout_after, 'canary')) as f:
	224	+ assert f.read() == 'alive\n'

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.