Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream
Commits:
-
f067397b
by Phil Dawson at 2018-11-19T09:19:24Z
3 changed files:
Changes:
| ... | ... | @@ -680,11 +680,13 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar): |
| 680 | 680 |
help='Create a tarball from the element\'s sources instead of a '
|
| 681 | 681 |
'file tree. If LOCATION is \'-\', the tarball will be dumped '
|
| 682 | 682 |
'to the standard output.')
|
| 683 |
+@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
|
|
| 683 | 684 |
@click.argument('element',
|
| 684 | 685 |
type=click.Path(readable=False))
|
| 685 | 686 |
@click.argument('location', type=click.Path())
|
| 686 | 687 |
@click.pass_obj
|
| 687 |
-def source_checkout(app, element, location, force, deps, fetch_, except_, tar_):
|
|
| 688 |
+def source_checkout(app, element, location, force, deps, fetch_, except_,
|
|
| 689 |
+ tar_, build_scripts):
|
|
| 688 | 690 |
"""Checkout sources of an element to the specified location
|
| 689 | 691 |
"""
|
| 690 | 692 |
with app.initialized():
|
| ... | ... | @@ -694,7 +696,8 @@ def source_checkout(app, element, location, force, deps, fetch_, except_, tar_): |
| 694 | 696 |
deps=deps,
|
| 695 | 697 |
fetch=fetch_,
|
| 696 | 698 |
except_targets=except_,
|
| 697 |
- tar=tar_)
|
|
| 699 |
+ tar=tar_,
|
|
| 700 |
+ include_build_scripts=build_scripts)
|
|
| 698 | 701 |
|
| 699 | 702 |
|
| 700 | 703 |
##################################################################
|
| ... | ... | @@ -433,7 +433,8 @@ class Stream(): |
| 433 | 433 |
deps='none',
|
| 434 | 434 |
fetch=False,
|
| 435 | 435 |
except_targets=(),
|
| 436 |
- tar=False):
|
|
| 436 |
+ tar=False,
|
|
| 437 |
+ include_build_scripts=False):
|
|
| 437 | 438 |
|
| 438 | 439 |
self._check_location_writable(location, force=force, tar=tar)
|
| 439 | 440 |
|
| ... | ... | @@ -449,14 +450,33 @@ class Stream(): |
| 449 | 450 |
|
| 450 | 451 |
# Stage all sources determined by scope
|
| 451 | 452 |
try:
|
| 452 |
- if tar:
|
|
| 453 |
- self._create_source_tarball(location, elements)
|
|
| 454 |
- else:
|
|
| 455 |
- self._write_element_sources(location, elements)
|
|
| 453 |
+ self._source_checkout(elements, location, force, deps, fetch,
|
|
| 454 |
+ except_targets, tar, include_build_scripts)
|
|
| 456 | 455 |
except BstError as e:
|
| 457 | 456 |
raise StreamError("Error while writing sources"
|
| 458 | 457 |
": '{}'".format(e), detail=e.detail, reason=e.reason) from e<
/span>
|
| 459 | 458 |
|
| 459 |
+ def _source_checkout(self, elements,
|
|
| 460 |
+ location=None,
|
|
| 461 |
+ force=False,
|
|
| 462 |
+ deps='none',
|
|
| 463 |
+ fetch=False,
|
|
| 464 |
+ except_targets=(),
|
|
| 465 |
+ tar=False,
|
|
| 466 |
+ include_build_scripts=False):
|
|
| 467 |
+ location = os.path.abspath(location)
|
|
| 468 |
+ if tar:
|
|
| 469 |
+ self._create_source_tarball(location, elements, include_build_scripts)
|
|
| 470 |
+ else:
|
|
| 471 |
+ self._write_element_sources(location, elements)
|
|
| 472 |
+ if include_build_scripts:
|
|
| 473 |
+ self._write_build_scripts(location, elements)
|
|
| 474 |
+ |
|
| 475 |
+ def _write_build_scripts(self, location, elements):
|
|
| 476 |
+ for element in elements:
|
|
| 477 |
+ self._write_element_script(location, element)
|
|
| 478 |
+ self._write_master_build_script(location, elements)
|
|
| 479 |
+ |
|
| 460 | 480 |
# workspace_open
|
| 461 | 481 |
#
|
| 462 | 482 |
# Open a project workspace
|
| ... | ... | @@ -741,7 +761,7 @@ class Stream(): |
| 741 | 761 |
]
|
| 742 | 762 |
|
| 743 | 763 |
self._write_element_sources(os.path.join(tempdir, "source"), elements)
|
| 744 |
- self._write_build_script(tempdir, elements)
|
|
| 764 |
+ self._write_master_build_script(tempdir, elements)
|
|
| 745 | 765 |
self._collect_sources(tempdir, tar_location,
|
| 746 | 766 |
target.normal_name, compression)
|
| 747 | 767 |
|
| ... | ... | @@ -1142,16 +1162,29 @@ class Stream(): |
| 1142 | 1162 |
element._stage_sources_at(element_source_dir)
|
| 1143 | 1163 |
|
| 1144 | 1164 |
# Create a tarball containing the sources of each element in elements
|
| 1145 |
- def _create_source_tarball(self, directory, elements):
|
|
| 1146 |
- with tarfile.open(name=directory, mode='w') as tf:
|
|
| 1147 |
- with TemporaryDirectory() as tmpdir:
|
|
| 1148 |
- self._write_element_sources(tmpdir, elements)
|
|
| 1149 |
- for item in os.listdir(tmpdir):
|
|
| 1150 |
- file_to_add = os.path.join(tmpdir, item)
|
|
| 1165 |
+ def _create_source_tarball(self, tar_name, elements, include_build_scripts):
|
|
| 1166 |
+ # Stage sources into a temporary directory then create a tarball from this
|
|
| 1167 |
+ with TemporaryDirectory() as tmpdir:
|
|
| 1168 |
+ self._write_element_sources(tmpdir, elements)
|
|
| 1169 |
+ if include_build_scripts:
|
|
| 1170 |
+ self._write_build_scripts(tmpdir, elements)
|
|
| 1171 |
+ self._create_tar_from_directory(tar_name, tmpdir)
|
|
| 1172 |
+ |
|
| 1173 |
+ # Create a tarball from the content of directory
|
|
| 1174 |
+ def _create_tar_from_directory(self, tar_name, directory):
|
|
| 1175 |
+ try:
|
|
| 1176 |
+ with tarfile.open(name=tar_name, mode='w') as tf:
|
|
| 1177 |
+ for item in os.listdir(str(directory)):
|
|
| 1178 |
+ file_to_add = os.path.join(directory, item)
|
|
| 1151 | 1179 |
tf.add(file_to_add, arcname=item)
|
| 1180 |
+ except OSError as e:
|
|
| 1181 |
+ # If we have a partially constructed tar file, clean up after ourselves
|
|
| 1182 |
+ if os.path.exists(tar_name):
|
|
| 1183 |
+ os.remove(tar_name)
|
|
| 1184 |
+ raise StreamError("Failed to create tar archieve: {}".format(e)) from e
|
|
| 1152 | 1185 |
|
| 1153 | 1186 |
# Write a master build script to the sandbox
|
| 1154 |
- def _write_build_script(self, directory, elements):
|
|
| 1187 |
+ def _write_master_build_script(self, directory, elements):
|
|
| 1155 | 1188 |
|
| 1156 | 1189 |
module_string = ""
|
| 1157 | 1190 |
for element in elements:
|
| ... | ... | @@ -154,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch): |
| 154 | 154 |
assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
|
| 155 | 155 |
else:
|
| 156 | 156 |
result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
|
| 157 |
+ |
|
| 158 |
+ |
|
| 159 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
| 160 |
+def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
|
|
| 161 |
+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 162 |
+ element_name = 'source-bundle/source-bundle-hello.bst'
|
|
| 163 |
+ normal_name = 'source-bundle-source-bundle-hello'
|
|
| 164 |
+ checkout = os.path.join(str(tmpdir), 'source-checkout')
|
|
| 165 |
+ |
|
| 166 |
+ args = ['source-checkout', '--include-build-scripts', element_name, checkout]
|
|
| 167 |
+ result = cli.run(project=project_path, args=args)
|
|
| 168 |
+ result.assert_success()
|
|
| 169 |
+ |
|
| 170 |
+ # There sould be a script for each element (just one in this case) and a top level build script
|
|
| 171 |
+ expected_scripts = ['build.sh', 'build-' + normal_name]
|
|
| 172 |
+ for script in expected_scripts:
|
|
| 173 |
+ assert script in os.listdir(checkout)
|
|
| 174 |
+ |
|
| 175 |
+ |
|
| 176 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
| 177 |
+def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
|
|
| 178 |
+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 179 |
+ element_name = 'source-bundle/source-bundle-hello.bst'
|
|
| 180 |
+ normal_name = 'source-bundle-source-bundle-hello'
|
|
| 181 |
+ tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
|
|
| 182 |
+ |
|
| 183 |
+ args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
|
|
| 184 |
+ result = cli.run(project=project_path, args=args)
|
|
| 185 |
+ result.assert_success()
|
|
| 186 |
+ |
|
| 187 |
+ expected_scripts = ['build.sh', 'build-' + normal_name]
|
|
| 188 |
+ |
|
| 189 |
+ with tarfile.open(tar_file, 'r') as tf:
|
|
| 190 |
+ for script in expected_scripts:
|
|
| 191 |
+ assert script in tf.getnames()
|