[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 2 commits: Add option to source-checkout to generate build scripts

[Phil Dawson <gitlab mg gitlab com>]

Title: GitLab

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

• 3ff77da9
  by Phil Dawson at 2018-11-19T09:46:29Z

  Add option to source-checkout to generate build scripts
  

• 64593863
  by Phil Dawson at 2018-11-19T09:46:29Z

  Remove source bundle command
  
  This is part of the work towards #672
  

4 changed files:

• buildstream/_frontend/cli.py
• buildstream/_stream.py
• − tests/frontend/source_bundle.py
• tests/frontend/source_checkout.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -680,11 +680,13 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
680	680	help='Create a tarball from the element\'s sources instead of a '
681	681	'file tree. If LOCATION is \'-\', the tarball will be dumped '
682	682	'to the standard output.')
	683	+@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
683	684	@click.argument('element',
684	685	type=click.Path(readable=False))
685	686	@click.argument('location', type=click.Path())
686	687	@click.pass_obj
687		-def source_checkout(app, element, location, force, deps, fetch_, except_, tar_):
	688	+def source_checkout(app, element, location, force, deps, fetch_, except_,
	689	+ tar_, build_scripts):
688	690	"""Checkout sources of an element to the specified location
689	691	"""
690	692	with app.initialized():
...	...	@@ -694,7 +696,8 @@ def source_checkout(app, element, location, force, deps, fetch_, except_, tar_):
694	696	deps=deps,
695	697	fetch=fetch_,
696	698	except_targets=except_,
697		- tar=tar_)
	699	+ tar=tar_,
	700	+ include_build_scripts=build_scripts)
698	701
699	702
700	703	##################################################################
...	...	@@ -833,34 +836,3 @@ def workspace_list(app):
833	836
834	837	with app.initialized():
835	838	app.stream.workspace_list()
836		-
837		-
838		-##################################################################
839		-# Source Bundle Command #
840		-##################################################################
841		-@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
842		-@click.option('--except', 'except_', multiple=True,
843		- type=click.Path(readable=False),
844		- help="Elements to except from the tarball")
845		-@click.option('--compression', default='gz',
846		- type=click.Choice(['none', 'gz', 'bz2', 'xz']),
847		- help="Compress the tar file using the given algorithm.")
848		-@click.option('--track', 'track_', default=False, is_flag=True,
849		- help="Track new source references before bundling")
850		-@click.option('--force', '-f', default=False, is_flag=True,
851		- help="Overwrite an existing tarball")
852		-@click.option('--directory', default=os.getcwd(),
853		- help="The directory to write the tarball to")
854		-@click.argument('element',
855		- type=click.Path(readable=False))
856		-@click.pass_obj
857		-def source_bundle(app, element, force, directory,
858		- track_, compression, except_):
859		- """Produce a source bundle to be manually executed
860		- """
861		- with app.initialized():
862		- app.stream.source_bundle(element, directory,
863		- track_first=track_,
864		- force=force,
865		- compression=compression,
866		- except_targets=except_)

buildstream/_stream.py

---

...	...	@@ -433,7 +433,8 @@ class Stream():
433	433	deps='none',
434	434	fetch=False,
435	435	except_targets=(),
436		- tar=False):
	436	+ tar=False,
	437	+ include_build_scripts=False):
437	438
438	439	self._check_location_writable(location, force=force, tar=tar)
439	440
...	...	@@ -449,14 +450,11 @@ class Stream():
449	450
450	451	# Stage all sources determined by scope
451	452	try:
452		- if tar:
453		- self._create_source_tarball(location, elements)
454		- else:
455		- self._write_element_sources(location, elements)
	453	+ self._source_checkout(elements, location, force, deps, fetch,
	454	+ except_targets, tar, include_build_scripts)
456	455	except BstError as e:
457	456	raise StreamError("Error while writing sources"
458	457	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
459		-
460	458	# workspace_open
461	459	#
462	460	# Open a project workspace
...	...	@@ -664,87 +662,6 @@ class Stream():
664	662	'workspaces': workspaces
665	663	})
666	664
667		- # source_bundle()
668		- #
669		- # Create a host buildable tarball bundle for the given target.
670		- #
671		- # Args:
672		- # target (str): The target element to bundle
673		- # directory (str): The directory to output the tarball
674		- # track_first (bool): Track new source references before bundling
675		- # compression (str): The compression type to use
676		- # force (bool): Overwrite an existing tarball
677		- #
678		- def source_bundle(self, target, directory, *,
679		- track_first=False,
680		- force=False,
681		- compression="gz",
682		- except_targets=()):
683		-
684		- if track_first:
685		- track_targets = (target,)
686		- else:
687		- track_targets = ()
688		-
689		- elements, track_elements = self._load((target,), track_targets,
690		- selection=PipelineSelection.ALL,
691		- except_targets=except_targets,
692		- track_selection=PipelineSelection.ALL,
693		- fetch_subprojects=True)
694		-
695		- # source-bundle only supports one target
696		- target = self.targets[0]
697		-
698		- self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
699		-
700		- # Find the correct filename for the compression algorithm
701		- tar_location = os.path.join(directory, target.normal_name + ".tar")
702		- if compression != "none":
703		- tar_location += "." + compression
704		-
705		- # Attempt writing a file to generate a good error message
706		- # early
707		- #
708		- # FIXME: A bit hackish
709		- try:
710		- open(tar_location, mode="x")
711		- os.remove(tar_location)
712		- except IOError as e:
713		- raise StreamError("Cannot write to {0}: {1}"
714		- .format(tar_location, e)) from e
715		-
716		- # Fetch and possibly track first
717		- #
718		- self._fetch(elements, track_elements=track_elements)
719		-
720		- # We don't use the scheduler for this as it is almost entirely IO
721		- # bound.
722		-
723		- # Create a temporary directory to build the source tree in
724		- builddir = self._context.builddir
725		- os.makedirs(builddir, exist_ok=True)
726		- prefix = "{}-".format(target.normal_name)
727		-
728		- with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
729		- source_directory = os.path.join(tempdir, 'source')
730		- try:
731		- os.makedirs(source_directory)
732		- except OSError as e:
733		- raise StreamError("Failed to create directory: {}"
734		- .format(e)) from e
735		-
736		- # Any elements that don't implement _write_script
737		- # should not be included in the later stages.
738		- elements = [
739		- element for element in elements
740		- if self._write_element_script(source_directory, element)
741		- ]
742		-
743		- self._write_element_sources(os.path.join(tempdir, "source"), elements)
744		- self._write_build_script(tempdir, elements)
745		- self._collect_sources(tempdir, tar_location,
746		- target.normal_name, compression)
747		-
748	665	# redirect_element_names()
749	666	#
750	667	# Takes a list of element names and returns a list where elements have been
...	...	@@ -1125,6 +1042,23 @@ class Stream():
1125	1042
1126	1043	sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
1127	1044
	1045	+ # Helper function for source_checkout()
	1046	+ def _source_checkout(self, elements,
	1047	+ location=None,
	1048	+ force=False,
	1049	+ deps='none',
	1050	+ fetch=False,
	1051	+ except_targets=(),
	1052	+ tar=False,
	1053	+ include_build_scripts=False):
	1054	+ location = os.path.abspath(location)
	1055	+ if tar:
	1056	+ self._create_source_tarball(location, elements, include_build_scripts)
	1057	+ else:
	1058	+ self._write_element_sources(location, elements)
	1059	+ if include_build_scripts:
	1060	+ self._write_build_scripts(location, elements)
	1061	+
1128	1062	# Write the element build script to the given directory
1129	1063	def _write_element_script(self, directory, element):
1130	1064	try:
...	...	@@ -1142,16 +1076,35 @@ class Stream():
1142	1076	element._stage_sources_at(element_source_dir)
1143	1077
1144	1078	# Create a tarball containing the sources of each element in elements
1145		- def _create_source_tarball(self, directory, elements):
1146		- with tarfile.open(name=directory, mode='w') as tf:
1147		- with TemporaryDirectory() as tmpdir:
1148		- self._write_element_sources(tmpdir, elements)
1149		- for item in os.listdir(tmpdir):
1150		- file_to_add = os.path.join(tmpdir, item)
	1079	+ def _create_source_tarball(self, tar_name, elements, include_build_scripts):
	1080	+ # Stage sources into a temporary directory then create a tarball from this
	1081	+ with TemporaryDirectory() as tmpdir:
	1082	+ self._write_element_sources(tmpdir, elements)
	1083	+ if include_build_scripts:
	1084	+ self._write_build_scripts(tmpdir, elements)
	1085	+ self._create_tar_from_directory(tar_name, tmpdir)
	1086	+
	1087	+ # Create a tarball from the content of directory
	1088	+ def _create_tar_from_directory(self, tar_name, directory):
	1089	+ try:
	1090	+ with tarfile.open(name=tar_name, mode='w') as tf:
	1091	+ for item in os.listdir(str(directory)):
	1092	+ file_to_add = os.path.join(directory, item)
1151	1093	tf.add(file_to_add, arcname=item)
	1094	+ except OSError as e:
	1095	+ # If we have a partially constructed tar file, clean up after ourselves
	1096	+ if os.path.exists(tar_name):
	1097	+ os.remove(tar_name)
	1098	+ raise StreamError("Failed to create tar archieve: {}".format(e)) from e
	1099	+
	1100	+ # Write all the build_scripts for elements in the directory location
	1101	+ def _write_build_scripts(self, location, elements):
	1102	+ for element in elements:
	1103	+ self._write_element_script(location, element)
	1104	+ self._write_master_build_script(location, elements)
1152	1105
1153	1106	# Write a master build script to the sandbox
1154		- def _write_build_script(self, directory, elements):
	1107	+ def _write_master_build_script(self, directory, elements):
1155	1108
1156	1109	module_string = ""
1157	1110	for element in elements:

tests/frontend/source_bundle.py deleted

---

1		-#
2		-# Copyright (C) 2018 Bloomberg Finance LP
3		-#
4		-# This program is free software; you can redistribute it and/or
5		-# modify it under the terms of the GNU Lesser General Public
6		-# License as published by the Free Software Foundation; either
7		-# version 2 of the License, or (at your option) any later version.
8		-#
9		-# This library is distributed in the hope that it will be useful,
10		-# but WITHOUT ANY WARRANTY; without even the implied warranty of
11		-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12		-# Lesser General Public License for more details.
13		-#
14		-# You should have received a copy of the GNU Lesser General Public
15		-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
16		-#
17		-# Authors: Chandan Singh <csingh43 bloomberg net>
18		-#
19		-
20		-import os
21		-import tarfile
22		-
23		-import pytest
24		-
25		-from tests.testutils import cli
26		-
27		-# Project directory
28		-DATA_DIR = os.path.join(
29		- os.path.dirname(os.path.realpath(__file__)),
30		- "project",
31		-)
32		-
33		-
34		-@pytest.mark.datafiles(DATA_DIR)
35		-def test_source_bundle(cli, tmpdir, datafiles):
36		- project_path = os.path.join(datafiles.dirname, datafiles.basename)
37		- element_name = 'source-bundle/source-bundle-hello.bst'
38		- normal_name = 'source-bundle-source-bundle-hello'
39		-
40		- # Verify that we can correctly produce a source-bundle
41		- args = ['source-bundle', element_name, '--directory', str(tmpdir)]
42		- result = cli.run(project=project_path, args=args)
43		- result.assert_success()
44		-
45		- # Verify that the source-bundle contains our sources and a build script
46		- with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
47		- assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
48		- assert os.path.join(normal_name, 'build.sh') in bundle.getnames()

tests/frontend/source_checkout.py

---

...	...	@@ -154,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
154	154	assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
155	155	else:
156	156	result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
	157	+
	158	+
	159	+@pytest.mark.datafiles(DATA_DIR)
	160	+def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
	161	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	162	+ element_name = 'source-bundle/source-bundle-hello.bst'
	163	+ normal_name = 'source-bundle-source-bundle-hello'
	164	+ checkout = os.path.join(str(tmpdir), 'source-checkout')
	165	+
	166	+ args = ['source-checkout', '--include-build-scripts', element_name, checkout]
	167	+ result = cli.run(project=project_path, args=args)
	168	+ result.assert_success()
	169	+
	170	+ # There sould be a script for each element (just one in this case) and a top level build script
	171	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	172	+ for script in expected_scripts:
	173	+ assert script in os.listdir(checkout)
	174	+
	175	+
	176	+@pytest.mark.datafiles(DATA_DIR)
	177	+def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
	178	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	179	+ element_name = 'source-bundle/source-bundle-hello.bst'
	180	+ normal_name = 'source-bundle-source-bundle-hello'
	181	+ tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
	182	+
	183	+ args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
	184	+ result = cli.run(project=project_path, args=args)
	185	+ result.assert_success()
	186	+
	187	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	188	+
	189	+ with tarfile.open(tar_file, 'r') as tf:
	190	+ for script in expected_scripts:
	191	+ assert script in tf.getnames()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.