[Notes] [Git][BuildStream/buildstream][jmac/make_writable_combination] 10 commits: plugins/sources/pip.py: Accomodate characters '-', '.', '_' for packages

[Jim MacArthur <gitlab mg gitlab com>]

Title: GitLab

Jim MacArthur pushed to branch jmac/make_writable_combination at BuildStream / buildstream

Commits:

• de59ebdb
  by ctolentino8 at 2018-11-02T16:41:54Z

  plugins/sources/pip.py: Accomodate characters '-','.','_' for packages
  

• 8d7cf806
  by ctolentino8 at 2018-11-02T16:41:54Z

  tests/integration/project/files: Remove pypi-repo
  

• 9c2f9bf7
  by Chandan Singh at 2018-11-02T17:09:46Z

  Merge branch 'chiaratolentino/fix-pip-source-regex' into 'master'
  
  plugins/sources/pip.py: Accomodate characters '-','.','_' for packages
  
  See merge request BuildStream/buildstream!914

• 3788e701
  by Jürg Billeter at 2018-11-03T11:52:00Z

  element.py: Pass bare_directory also to SandboxRemote()
  
  Fixes: 825963b5 ("element: Make "--sysroot" take a bare directory")
  

• 82e971ef
  by Jürg Billeter at 2018-11-05T11:33:20Z

  Merge branch 'juerg/remote-execution' into 'master'
  
  element.py: Pass bare_directory also to SandboxRemote()
  
  See merge request BuildStream/buildstream!928

• 62942bfd
  by Valentin David at 2018-11-05T12:14:20Z

  Use less fetchers in overnight build to work-around limitations from servers.
  
  Issue happens with sourceware.org server.
  
  Fixes #742
  

• 442da2f9
  by Javier Jardón at 2018-11-05T12:41:54Z

  Merge branch 'valentindavid/limit_fetchers_in_overnight_build' into 'master'
  
  Use less fetchers in overnight build to work-around limitations from servers.
  
  Closes #742
  
  See merge request BuildStream/buildstream!929

• 0993de53
  by Richard Maw at 2018-11-05T16:41:05Z

  dev-requirements: Update pytest dependency to 3.8
  
  "_frontend/app.py: Set correct element-path in interactive bst-init"
  added a dependency on the tmp_path test fixture, which was added by 3.8.
  

• be8f0a54
  by richardmaw-codethink at 2018-11-05T17:08:43Z

  Merge branch 'richardmaw/pytest-dep-update' into 'master'
  
  dev-requirements: Update pytest dependency to 3.8
  
  See merge request BuildStream/buildstream!930

• 4fe6d95d
  by Jim MacArthur at 2018-11-05T17:12:46Z

  element.py: Use _force_rmtree instead of custom code.
  
  There are arguments for and against using onerror - see issue #735
  for details.
  

12 changed files:

• .gitlab-ci.yml
• buildstream/element.py
• buildstream/plugins/sources/pip.py
• dev-requirements.txt
• tests/integration/pip_source.py
• − tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
• − tests/integration/project/files/pypi-repo/app2/index.html
• − tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz
• − tests/integration/project/files/pypi-repo/hellolib/index.html
• tests/sources/pip.py
• tests/testutils/__init__.py
• + tests/testutils/python_repo.py

Changes:

.gitlab-ci.yml

---

...	...	@@ -166,6 +166,12 @@ docs:
166	166	BST_EXT_REF: 1d6ab71151b93c8cbc0a91a36ffe9270f3b835f1 # 0.5.1
167	167	FD_SDK_REF: 88d7c22c2281b987faa02edd57df80d430eecf1f # 18.08.11-35-g88d7c22c
168	168	before_script:
	169	+ - |
	170	+ mkdir -p "${HOME}/.config"
	171	+ cat <<EOF >"${HOME}/.config/buildstream.conf"
	172	+ scheduler:
	173	+ fetchers: 2
	174	+ EOF
169	175	- (cd dist && ./unpack.sh && cd buildstream && pip3 install .)
170	176	- pip3 install --user -e ${BST_EXT_URL}@${BST_EXT_REF}#egg=bst_ext
171	177	- git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git

buildstream/element.py

---

...	...	@@ -1411,16 +1411,9 @@ class Element(Plugin):
1411	1411
1412	1412	finally:
1413	1413	# Staging may produce directories with less than 'rwx' permissions
1414		- # for the owner, which will break tempfile, so we need to use chmod
1415		- # occasionally.
1416		- def make_dir_writable(fn, path, excinfo):
1417		- os.chmod(os.path.dirname(path), 0o777)
1418		- if os.path.isdir(path):
1419		- os.rmdir(path)
1420		- else:
1421		- os.remove(path)
1422		- shutil.rmtree(temp_staging_directory, onerror=make_dir_writable)
1423		-
	1414	+ # for the owner, which breaks tempfile. _force_rmtree will deal
	1415	+ # with these.
	1416	+ utils._force_rmtree(temp_staging_directory)
1424	1417	# Ensure deterministic mtime of sources at build time
1425	1418	vdirectory.set_deterministic_mtime()
1426	1419	# Ensure deterministic owners of sources at build time
...	...	@@ -2180,6 +2173,7 @@ class Element(Plugin):
2180	2173	stderr=stderr,
2181	2174	config=config,
2182	2175	server_url=self.__remote_execution_url,
	2176	+ bare_directory=bare_directory,
2183	2177	allow_real_directory=False)
2184	2178	yield sandbox
2185	2179

buildstream/plugins/sources/pip.py

---

...	...	@@ -96,7 +96,7 @@ _PYTHON_VERSIONS = [
96	96	# Names of source distribution archives must be of the form
97	97	# '%{package-name}-%{version}.%{extension}'.
98	98	_SDIST_RE = re.compile(
99		- r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
	99	+ r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
100	100	re.IGNORECASE)
101	101
102	102
...	...	@@ -225,12 +225,27 @@ class PipSource(Source):
225	225	def _parse_sdist_names(self, basedir):
226	226	reqs = []
227	227	for f in os.listdir(basedir):
228		- pkg_match = _SDIST_RE.match(f)
229		- if pkg_match:
230		- reqs.append(pkg_match.groups())
	228	+ pkg = _match_package_name(f)
	229	+ if pkg is not None:
	230	+ reqs.append(pkg)
231	231
232	232	return sorted(reqs)
233	233
234	234
	235	+# Extract the package name and version of a source distribution
	236	+#
	237	+# Args:
	238	+# filename (str): Filename of the source distribution
	239	+#
	240	+# Returns:
	241	+# (tuple): A tuple of (package_name, version)
	242	+#
	243	+def _match_package_name(filename):
	244	+ pkg_match = _SDIST_RE.match(filename)
	245	+ if pkg_match is None:
	246	+ return None
	247	+ return pkg_match.groups()
	248	+
	249	+
235	250	def setup():
236	251	return PipSource

dev-requirements.txt

---

1	1	coverage == 4.4.0
2	2	pep8
3	3	pylint == 2.1.1
4		-pytest >= 3.7
	4	+pytest >= 3.8
5	5	pytest-cov >= 2.5.0
6	6	pytest-datafiles
7	7	pytest-env

tests/integration/pip_source.py

---

...	...	@@ -4,6 +4,7 @@ import pytest
4	4	from buildstream import _yaml
5	5
6	6	from tests.testutils import cli_integration as cli
	7	+from tests.testutils.python_repo import setup_pypi_repo
7	8	from tests.testutils.integration import assert_contains
8	9
9	10
...	...	@@ -17,12 +18,21 @@ DATA_DIR = os.path.join(
17	18
18	19
19	20	@pytest.mark.datafiles(DATA_DIR)
20		-def test_pip_source_import(cli, tmpdir, datafiles):
	21	+def test_pip_source_import(cli, tmpdir, datafiles, setup_pypi_repo):
21	22	project = os.path.join(datafiles.dirname, datafiles.basename)
22	23	checkout = os.path.join(cli.directory, 'checkout')
23	24	element_path = os.path.join(project, 'elements')
24	25	element_name = 'pip/hello.bst'
25	26
	27	+ # check that exotically named packages are imported correctly
	28	+ myreqs_packages = ['hellolib']
	29	+ packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
	30	+
	31	+ # create mock pypi repository
	32	+ pypi_repo = os.path.join(project, 'files', 'pypi-repo')
	33	+ os.makedirs(pypi_repo, exist_ok=True)
	34	+ setup_pypi_repo(myreqs_packages + packages, pypi_repo)
	35	+
26	36	element = {
27	37	'kind': 'import',
28	38	'sources': [
...	...	@@ -32,9 +42,9 @@ def test_pip_source_import(cli, tmpdir, datafiles):
32	42	},
33	43	{
34	44	'kind': 'pip',
35		- 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
	45	+ 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
36	46	'requirements-files': ['myreqs.txt'],
37		- 'packages': ['app2']
	47	+ 'packages': packages
38	48	}
39	49	]
40	50	}
...	...	@@ -51,16 +61,31 @@ def test_pip_source_import(cli, tmpdir, datafiles):
51	61	assert result.exit_code == 0
52	62
53	63	assert_contains(checkout, ['/.bst_pip_downloads',
54		- '/.bst_pip_downloads/HelloLib-0.1.tar.gz',
55		- '/.bst_pip_downloads/App2-0.1.tar.gz'])
	64	+ '/.bst_pip_downloads/hellolib-0.1.tar.gz',
	65	+ '/.bst_pip_downloads/app2-0.1.tar.gz',
	66	+ '/.bst_pip_downloads/app.3-0.1.tar.gz',
	67	+ '/.bst_pip_downloads/app-4-0.1.tar.gz',
	68	+ '/.bst_pip_downloads/app_5-0.1.tar.gz',
	69	+ '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
	70	+ '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
	71	+ '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
56	72
57	73
58	74	@pytest.mark.datafiles(DATA_DIR)
59		-def test_pip_source_build(cli, tmpdir, datafiles):
	75	+def test_pip_source_build(cli, tmpdir, datafiles, setup_pypi_repo):
60	76	project = os.path.join(datafiles.dirname, datafiles.basename)
61	77	element_path = os.path.join(project, 'elements')
62	78	element_name = 'pip/hello.bst'
63	79
	80	+ # check that exotically named packages are imported correctly
	81	+ myreqs_packages = ['hellolib']
	82	+ packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
	83	+
	84	+ # create mock pypi repository
	85	+ pypi_repo = os.path.join(project, 'files', 'pypi-repo')
	86	+ os.makedirs(pypi_repo, exist_ok=True)
	87	+ setup_pypi_repo(myreqs_packages + packages, pypi_repo)
	88	+
64	89	element = {
65	90	'kind': 'manual',
66	91	'depends': ['base.bst'],
...	...	@@ -71,16 +96,15 @@ def test_pip_source_build(cli, tmpdir, datafiles):
71	96	},
72	97	{
73	98	'kind': 'pip',
74		- 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
	99	+ 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
75	100	'requirements-files': ['myreqs.txt'],
76		- 'packages': ['app2']
	101	+ 'packages': packages
77	102	}
78	103	],
79	104	'config': {
80	105	'install-commands': [
81	106	'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz',
82		- 'chmod +x app1.py',
83		- 'install app1.py %{install-root}/usr/bin/'
	107	+ 'install app1.py %{install-root}/usr/bin/'
84	108	]
85	109	}
86	110	}
...	...	@@ -95,5 +119,4 @@ def test_pip_source_build(cli, tmpdir, datafiles):
95	119
96	120	result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
97	121	assert result.exit_code == 0
98		- assert result.output == """Hello App1!
99		-"""
	122	+ assert result.output == "Hello App1! This is hellolib\n"

tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz deleted

---

No preview for this file type

tests/integration/project/files/pypi-repo/app2/index.html deleted

---

1		-<html>
2		- <head>
3		- <title>Links for app1</title>
4		- </head>
5		- <body>
6		- <a href="">'App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
7		- </body>
8		-</html>

tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz deleted

---

No preview for this file type

tests/integration/project/files/pypi-repo/hellolib/index.html deleted

---

1		-<html>
2		- <head>
3		- <title>Links for app1</title>
4		- </head>
5		- <body>
6		- <a href="">'HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
7		- </body>
8		-</html>

tests/sources/pip.py

---

...	...	@@ -3,6 +3,7 @@ import pytest
3	3
4	4	from buildstream._exceptions import ErrorDomain
5	5	from buildstream import _yaml
	6	+from buildstream.plugins.sources.pip import _match_package_name
6	7	from tests.testutils import cli
7	8
8	9	DATA_DIR = os.path.join(
...	...	@@ -45,3 +46,22 @@ def test_no_packages(cli, tmpdir, datafiles):
45	46	'show', 'target.bst'
46	47	])
47	48	result.assert_main_error(ErrorDomain.SOURCE, None)
	49	+
	50	+
	51	+# Test that pip source parses tar ball names correctly for the ref
	52	+@pytest.mark.parametrize(
	53	+ 'tarball, expected_name, expected_version',
	54	+ [
	55	+ ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'),
	56	+ ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'),
	57	+ ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'),
	58	+ ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'),
	59	+ ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'),
	60	+ ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'),
	61	+ ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'),
	62	+ ('shortversion-1.0.tar.gz', 'shortversion', '1.0'),
	63	+ ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4')
	64	+ ])
	65	+def test_match_package_name(tarball, expected_name, expected_version):
	66	+ name, version = _match_package_name(tarball)
	67	+ assert (expected_name, expected_version) == (name, version)

tests/testutils/__init__.py

---

...	...	@@ -29,3 +29,4 @@ from .artifactshare import create_artifact_share
29	29	from .element_generators import create_element_size, update_element_size
30	30	from .junction import generate_junction
31	31	from .runner_integration import wait_for_cache_granularity
	32	+from .python_repo import setup_pypi_repo

tests/testutils/python_repo.py

---

	1	+from setuptools.sandbox import run_setup
	2	+import os
	3	+import pytest
	4	+import re
	5	+import shutil
	6	+
	7	+
	8	+SETUP_TEMPLATE = '''\
	9	+from setuptools import setup
	10	+
	11	+setup(
	12	+ name='{name}',
	13	+ version='{version}',
	14	+ description='{name}',
	15	+ packages=['{pkgdirname}'],
	16	+ entry_points={{
	17	+ 'console_scripts': [
	18	+ '{pkgdirname}={pkgdirname}:main'
	19	+ ]
	20	+ }}
	21	+)
	22	+'''
	23	+
	24	+# All packages generated via generate_pip_package will have the functions below
	25	+INIT_TEMPLATE = '''\
	26	+def main():
	27	+ print('This is {name}')
	28	+
	29	+def hello(actor='world'):
	30	+ print('Hello {{}}! This is {name}'.format(actor))
	31	+'''
	32	+
	33	+HTML_TEMPLATE = '''\
	34	+<html>
	35	+ <head>
	36	+ <title>Links for {name}</title>
	37	+ </head>
	38	+ <body>
	39	+ <a href=''>{name}-{version}.tar.gz</a><br />
	40	+ </body>
	41	+</html>
	42	+'''
	43	+
	44	+
	45	+# Creates a simple python source distribution and copies this into a specified
	46	+# directory which is to serve as a mock python repository
	47	+#
	48	+# Args:
	49	+# tmpdir (str): Directory in which the source files will be created
	50	+# pypi (str): Directory serving as a mock python repository
	51	+# name (str): The name of the package to be created
	52	+# version (str): The version of the package to be created
	53	+#
	54	+# Returns:
	55	+# None
	56	+#
	57	+def generate_pip_package(tmpdir, pypi, name, version='0.1'):
	58	+ # check if package already exists in pypi
	59	+ pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
	60	+ if os.path.exists(pypi_package):
	61	+ return
	62	+
	63	+ # create the package source files in tmpdir resulting in a directory
	64	+ # tree resembling the following structure:
	65	+ #
	66	+ # tmpdir
	67	+ # |-- setup.py
	68	+ # `-- package
	69	+ # `-- __init__.py
	70	+ #
	71	+ setup_file = os.path.join(tmpdir, 'setup.py')
	72	+ pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name)
	73	+ with open(setup_file, 'w') as f:
	74	+ f.write(
	75	+ SETUP_TEMPLATE.format(
	76	+ name=name,
	77	+ version=version,
	78	+ pkgdirname=pkgdirname
	79	+ )
	80	+ )
	81	+ os.chmod(setup_file, 0o755)
	82	+
	83	+ package = os.path.join(tmpdir, pkgdirname)
	84	+ os.makedirs(package)
	85	+
	86	+ main_file = os.path.join(package, '__init__.py')
	87	+ with open(main_file, 'w') as f:
	88	+ f.write(INIT_TEMPLATE.format(name=name))
	89	+ os.chmod(main_file, 0o644)
	90	+
	91	+ run_setup(setup_file, ['sdist'])
	92	+
	93	+ # create directory for this package in pypi resulting in a directory
	94	+ # tree resembling the following structure:
	95	+ #
	96	+ # pypi
	97	+ # `-- pypi_package
	98	+ # |-- index.html
	99	+ # `-- foo-0.1.tar.gz
	100	+ #
	101	+ os.makedirs(pypi_package)
	102	+
	103	+ # add an index html page
	104	+ index_html = os.path.join(pypi_package, 'index.html')
	105	+ with open(index_html, 'w') as f:
	106	+ f.write(HTML_TEMPLATE.format(name=name, version=version))
	107	+
	108	+ # copy generated tarfile to pypi package
	109	+ dist_dir = os.path.join(tmpdir, 'dist')
	110	+ for tar in os.listdir(dist_dir):
	111	+ tarpath = os.path.join(dist_dir, tar)
	112	+ shutil.copy(tarpath, pypi_package)
	113	+
	114	+
	115	+@pytest.fixture
	116	+def setup_pypi_repo(tmpdir):
	117	+ def create_pkgdir(package):
	118	+ pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package)
	119	+ pkgdir = os.path.join(str(tmpdir), pkgdirname)
	120	+ os.makedirs(pkgdir)
	121	+ return pkgdir
	122	+
	123	+ def add_packages(packages, pypi_repo):
	124	+ for package in packages:
	125	+ pkgdir = create_pkgdir(package)
	126	+ generate_pip_package(pkgdir, pypi_repo, package)
	127	+
	128	+ return add_packages

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.