[Notes] [Git][BuildStream/buildstream][tiagogomes/issue-577-backport] 3 commits: Fix ostree repository mirroring

[Tiago Gomes <gitlab mg gitlab com>]

Title: GitLab

Tiago Gomes pushed to branch tiagogomes/issue-577-backport at BuildStream / buildstream

Commits:

• de73cc0b
  by Valentin David at 2018-08-16T13:37:06Z

  Fix ostree repository mirroring
  
  Ostree mirrors were not sharing the same local repository, so it was
  impossible the request refs from the right local repository when data
  was fetched from a mirror rather than upstream.
  
  Instead of having several repository with one remote each, we now
  have one repository with several remotes.
  
  This fixes #538.
  

• 43f4ffba
  by Valentin David at 2018-08-16T14:36:49Z

  Merge branch 'valentindavid/fallback_mirror_ostree-1.2' into 'bst-1.2'
  
  Fix ostree repository mirroring
  
  See merge request BuildStream/buildstream!667

• f15ff456
  by Tiago Gomes at 2018-08-16T15:00:34Z

  cascache: use errno module
  
  os.errno does no longer work with Python 3.7
  
  Closes #577.
  

3 changed files:

• buildstream/_artifactcache/cascache.py
• buildstream/plugins/sources/ostree.py
• tests/frontend/mirror.py

Changes:

buildstream/_artifactcache/cascache.py

---

...	...	@@ -24,6 +24,7 @@ import os
24	24	import signal
25	25	import stat
26	26	import tempfile
	27	+import errno
27	28	from urllib.parse import urlparse
28	29
29	30	import grpc
...	...	@@ -81,7 +82,8 @@ class CASCache(ArtifactCache):
81	82
82	83	tree = self.resolve_ref(ref, update_mtime=True)
83	84
84		- dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
	85	+ dest = os.path.join(self.extractdir, element._get_project().name,
	86	+ element.normal_name, tree.hash)
85	87	if os.path.isdir(dest):
86	88	# artifact has already been extracted
87	89	return dest
...	...	@@ -99,7 +101,7 @@ class CASCache(ArtifactCache):
99	101	#
100	102	# If rename fails with these errors, another process beat
101	103	# us to it so just ignore.
102		- if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]:
	104	+ if e.errno not in [errno.ENOTEMPTY, errno.EEXIST]:
103	105	raise ArtifactError("Failed to extract artifact for ref '{}': {}"
104	106	.format(ref, e)) from e
105	107

buildstream/plugins/sources/ostree.py

---

...	...	@@ -71,7 +71,7 @@ class OSTreeSource(Source):
71	71	self.ref = self.node_get_member(node, str, 'ref', None)
72	72	self.tracking = self.node_get_member(node, str, 'track', None)
73	73	self.mirror = os.path.join(self.get_mirror_directory(),
74		- utils.url_directory_name(self.url))
	74	+ utils.url_directory_name(self.original_url))
75	75
76	76	# (optional) Not all repos are signed. But if they are, get the gpg key
77	77	self.gpg_key_path = None
...	...	@@ -104,10 +104,11 @@ class OSTreeSource(Source):
104	104	return None
105	105
106	106	self.ensure()
	107	+ remote_name = self.ensure_remote(self.url)
107	108	with self.timed_activity("Fetching tracking ref '{}' from origin: {}"
108	109	.format(self.tracking, self.url)):
109	110	try:
110		- _ostree.fetch(self.repo, ref=self.tracking, progress=self.progress)
	111	+ _ostree.fetch(self.repo, remote=remote_name, ref=self.tracking, progress=self.progress)
111	112	except OSTreeError as e:
112	113	raise SourceError("{}: Failed to fetch tracking ref '{}' from origin {}\n\n{}"
113	114	.format(self, self.tracking, self.url, e)) from e
...	...	@@ -116,11 +117,12 @@ class OSTreeSource(Source):
116	117
117	118	def fetch(self):
118	119	self.ensure()
	120	+ remote_name = self.ensure_remote(self.url)
119	121	if not _ostree.exists(self.repo, self.ref):
120	122	with self.timed_activity("Fetching remote ref: {} from origin: {}"
121	123	.format(self.ref, self.url)):
122	124	try:
123		- _ostree.fetch(self.repo, ref=self.ref, progress=self.progress)
	125	+ _ostree.fetch(self.repo, remote=remote_name, ref=self.ref, progress=self.progress)
124	126	except OSTreeError as e:
125	127	raise SourceError("{}: Failed to fetch ref '{}' from origin: {}\n\n{}"
126	128	.format(self, self.ref, self.url, e)) from e
...	...	@@ -171,14 +173,22 @@ class OSTreeSource(Source):
171	173	self.status("Creating local mirror for {}".format(self.url))
172	174
173	175	self.repo = _ostree.ensure(self.mirror, True)
174		- gpg_key = None
175		- if self.gpg_key_path:
176		- gpg_key = 'file://' + self.gpg_key_path
177	176
178		- try:
179		- _ostree.configure_remote(self.repo, "origin", self.url, key_url=gpg_key)
180		- except OSTreeError as e:
181		- raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
	177	+ def ensure_remote(self, url):
	178	+ if self.original_url == self.url:
	179	+ remote_name = 'origin'
	180	+ else:
	181	+ remote_name = utils.url_directory_name(url)
	182	+
	183	+ gpg_key = None
	184	+ if self.gpg_key_path:
	185	+ gpg_key = 'file://' + self.gpg_key_path
	186	+
	187	+ try:
	188	+ _ostree.configure_remote(self.repo, remote_name, url, key_url=gpg_key)
	189	+ except OSTreeError as e:
	190	+ raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
	191	+ return remote_name
182	192
183	193	def progress(self, percent, message):
184	194	self.status(message)

tests/frontend/mirror.py

---

...	...	@@ -466,10 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
466	466	@pytest.mark.datafiles(DATA_DIR)
467	467	@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
468	468	def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
469		- if kind == 'ostree':
470		- # FIXME: Mirroring fallback fails with ostree
471		- pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
472		-
473	469	bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
474	470	upstream_repodir = os.path.join(str(tmpdir), 'upstream')
475	471	mirror_repodir = os.path.join(str(tmpdir), 'mirror')

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.