GNOME.org
 

[Notes] [Git][BuildStream/buildstream][willsalmon/shellBuildTrees] Basic options for shell --build to use buildtrees

Title: GitLab

Will Salmon pushed to branch willsalmon/shellBuildTrees at BuildStream / buildstream

Commits:

6 changed files:

Changes:

  • buildstream/_frontend/cli.py
    ... ... @@ -527,11 +527,14 @@ def show(app, elements, deps, except_, order, format_):
    527 527 
                   help="Mount a file or directory into the sandbox")
    528 528 
     @click.option('--isolate', is_flag=True, default=False,
    529 529 
                   help='Create an isolated build sandbox')
    530 
    +@click.option('--use-buildtree', '-t', 'cli_buildtree', type=click.Choice(['ask', 'if-available', 'always', 'never']),
    531 
    +              default='ask',
    532 
    +              help='Defaults to ask but if set to always the function will fail if a build tree is not available')
    530 533 
     @click.argument('element', required=False,
    531 534 
                     type=click.Path(readable=False))
    532 535 
     @click.argument('command', type=click.STRING, nargs=-1)
    533 536 
     @click.pass_obj
    534 
    -def shell(app, element, sysroot, mount, isolate, build_, command):
    537 
    +def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command):
    535 538 
         """Run a command in the target element's sandbox environment
    536 539
    537 540 
         This will stage a temporary sysroot for running the target
    ... ... @@ -557,6 +560,8 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
    557 560 
         else:
    558 561 
             scope = Scope.RUN
    559 562
    563 
    +    use_buildtree = False
    564 
    +
    560 565 
         with app.initialized():
    561 566 
             if not element:
    562 567 
                 element = app.context.guess_element()
    ... ... @@ -570,12 +575,30 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
    570 575 
                 HostMount(path, host_path)
    571 576 
                 for host_path, path in mount
    572 577 
             ]
    578 
    +
    579 
    +        cached = element._cached_buildtree()
    580 
    +        if cli_buildtree == "always":
    581 
    +            if cached:
    582 
    +                use_buildtree = True
    583 
    +            else:
    584 
    +                raise AppError("No buildtree is cached but the use buildtree option was specified")
    585 
    +        elif cli_buildtree == "never":
    586 
    +            pass
    587 
    +        elif cli_buildtree == "if-available":
    588 
    +            use_buildtree = cached
    589 
    +        else:
    590 
    +            if app.interactive and cached:
    591 
    +                use_buildtree = bool(click.confirm('Do you want to use the cached buildtree?'))
    592 
    +        if use_buildtree and not element._cached_success():
    593 
    +            click.echo("Warning: using a buildtree from a failed build.")
    594 
    +
    573 595 
             try:
    574 596 
                 exitcode = app.stream.shell(element, scope, prompt,
    575 597 
                                             directory=sysroot,
    576 598 
                                             mounts=mounts,
    577 599 
                                             isolate=isolate,
    578 
    -                                        command=command)
    600 
    +                                        command=command,
    601 
    +                                        usebuildtree=use_buildtree)
    579 602 
             except BstError as e:
    580 603 
                 raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e
    581 604

  • buildstream/_stream.py
    ... ... @@ -124,6 +124,7 @@ class Stream():
    124 124 
         #    mounts (list of HostMount): Additional directories to mount into the sandbox
    125 125 
         #    isolate (bool): Whether to isolate the environment like we do in builds
    126 126 
         #    command (list): An argv to launch in the sandbox, or None
    127 
    +    #    usebuildtree (bool): Wheather to use a buildtree as the source.
    127 128 
         #
    128 129 
         # Returns:
    129 130 
         #    (int): The exit code of the launched shell
    ... ... @@ -132,7 +133,8 @@ class Stream():
    132 133 
                   directory=None,
    133 134 
                   mounts=None,
    134 135 
                   isolate=False,
    135 
    -              command=None):
    136 
    +              command=None,
    137 
    +              usebuildtree=False):
    136 138
    137 139 
             # Assert we have everything we need built, unless the directory is specified
    138 140 
             # in which case we just blindly trust the directory, using the element
    ... ... @@ -147,7 +149,8 @@ class Stream():
    147 149 
                     raise StreamError("Elements need to be built or downloaded before staging a shell environment",
    148 150 
                                       detail="\n".join(missing_deps))
    149 151
    150 
    -        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
    152 
    +        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command,
    153 
    +                              usebuildtree=usebuildtree)
    151 154
    152 155 
         # build()
    153 156 
         #

  • buildstream/element.py
    ... ... @@ -1338,11 +1338,12 @@ class Element(Plugin):
    1338 1338 
         # is used to stage things by the `bst checkout` codepath
    1339 1339 
         #
    1340 1340 
         @contextmanager
    1341 
    -    def _prepare_sandbox(self, scope, directory, shell=False, integrate=True):
    1341 
    +    def _prepare_sandbox(self, scope, directory, shell=False, integrate=True, usebuildtree=False):
    1342 1342 
             # bst shell and bst checkout require a local sandbox.
    1343 1343 
             bare_directory = True if directory else False
    1344 1344 
             with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
    1345 1345 
                                 bare_directory=bare_directory) as sandbox:
    1346 
    +            sandbox._usebuildtree = usebuildtree
    1346 1347
    1347 1348 
                 # Configure always comes first, and we need it.
    1348 1349 
                 self.__configure_sandbox(sandbox)
    ... ... @@ -1386,7 +1387,7 @@ class Element(Plugin):
    1386 1387 
             # Stage all sources that need to be copied
    1387 1388 
             sandbox_vroot = sandbox.get_virtual_directory()
    1388 1389 
             host_vdirectory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
    1389 
    -        self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces)
    1390 
    +        self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces, usebuildtree=sandbox._usebuildtree)
    1390 1391
    1391 1392 
         # _stage_sources_at():
    1392 1393 
         #
    ... ... @@ -1395,10 +1396,10 @@ class Element(Plugin):
    1395 1396 
         # Args:
    1396 1397 
         #     vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into.
    1397 1398 
         #     mount_workspaces (bool): mount workspaces if True, copy otherwise
    1399 
    +    #     usebuildtree (bool): use a the elements build tree as its source.
    1398 1400 
         #
    1399 
    -    def _stage_sources_at(self, vdirectory, mount_workspaces=True):
    1401 
    +    def _stage_sources_at(self, vdirectory, mount_workspaces=True, usebuildtree=False):
    1400 1402 
             with self.timed_activity("Staging sources", silent_nested=True):
    1401 
    -
    1402 1403 
                 if not isinstance(vdirectory, Directory):
    1403 1404 
                     vdirectory = FileBasedDirectory(vdirectory)
    1404 1405 
                 if not vdirectory.is_empty():
    ... ... @@ -1420,7 +1421,7 @@ class Element(Plugin):
    1420 1421 
                                                      .format(workspace.get_absolute_path())):
    1421 1422 
                                 workspace.stage(temp_staging_directory)
    1422 1423 
                     # Check if we have a cached buildtree to use
    1423 
    -                elif self._cached_buildtree():
    1424 
    +                elif usebuildtree:
    1424 1425 
                         artifact_base, _ = self.__extract()
    1425 1426 
                         import_dir = os.path.join(artifact_base, 'buildtree')
    1426 1427 
                     else:
    ... ... @@ -1850,13 +1851,15 @@ class Element(Plugin):
    1850 1851 
         #    isolate (bool): Whether to isolate the environment like we do in builds
    1851 1852 
         #    prompt (str): A suitable prompt string for PS1
    1852 1853 
         #    command (list): An argv to launch in the sandbox
    1854 
    +    #    usebuildtree (bool): Use the buildtree as its source
    1853 1855 
         #
    1854 1856 
         # Returns: Exit code
    1855 1857 
         #
    1856 1858 
         # If directory is not specified, one will be staged using scope
    1857 
    -    def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
    1859 
    +    def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None,
    1860 
    +               usebuildtree=False):
    1858 1861
    1859 
    -        with self._prepare_sandbox(scope, directory, shell=True) as sandbox:
    1862 
    +        with self._prepare_sandbox(scope, directory, shell=True, usebuildtree=usebuildtree) as sandbox:
    1860 1863 
                 environment = self.get_environment()
    1861 1864 
                 environment = copy.copy(environment)
    1862 1865 
                 flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY

  • buildstream/sandbox/sandbox.py
    ... ... @@ -116,6 +116,7 @@ class Sandbox():
    116 116 
             self.__env = None
    117 117 
             self.__mount_sources = {}
    118 118 
             self.__allow_real_directory = kwargs['allow_real_directory']
    119 
    +        self.usebuildtree = False
    119 120
    120 121 
             # Plugin ID for logging
    121 122 
             plugin = kwargs.get('plugin', None)
    ... ... @@ -146,6 +147,7 @@ class Sandbox():
    146 147
    147 148 
             self._output_directory = None
    148 149 
             self._vdir = None
    150 
    +        self._usebuildtree = False
    149 151
    150 152 
             # This is set if anyone requests access to the underlying
    151 153 
             # directory via get_directory.

  • tests/integration/build-tree.py
    ... ... @@ -19,7 +19,9 @@ DATA_DIR = os.path.join(
    19 19 
     @pytest.mark.datafiles(DATA_DIR)
    20 20 
     @pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    21 21 
     def test_buildtree_staged(cli_integration, tmpdir, datafiles):
    22 
    -    # i.e. tests that cached build trees are staged by `bst shell --build`
    22 
    +    # We can only test the non interacitve case
    23 
    +    # The non interactive case defaults to not using buildtrees
    24 
    +    # for `bst shell --build`
    23 25 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    24 26 
         element_name = 'build-shell/buildtree.bst'
    25 27
    ... ... @@ -27,15 +29,67 @@ def test_buildtree_staged(cli_integration, tmpdir, datafiles):
    27 29 
         res.assert_success()
    28 30
    29 31 
         res = cli_integration.run(project=project, args=[
    30 
    -        'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
    32 
    +        'shell', '--build', element_name, '--', 'cat', 'test'
    33 
    +    ])
    34 
    +    res.assert_shell_error()
    35 
    +
    36 
    +
    37 
    +@pytest.mark.datafiles(DATA_DIR)
    38 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    39 
    +def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles):
    40 
    +    # Test that if we ask for a build tree it is there.
    41 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    42 
    +    element_name = 'build-shell/buildtree.bst'
    43 
    +
    44 
    +    res = cli_integration.run(project=project, args=['build', element_name])
    45 
    +    res.assert_success()
    46 
    +
    47 
    +    res = cli_integration.run(project=project, args=[
    48 
    +        'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
    49 
    +    ])
    50 
    +    res.assert_success()
    51 
    +    assert 'Hi' in res.output
    52 
    +
    53 
    +
    54 
    +@pytest.mark.datafiles(DATA_DIR)
    55 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    56 
    +def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles):
    57 
    +    # Test that a build tree can be correctly detected.
    58 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    59 
    +    element_name = 'build-shell/buildtree.bst'
    60 
    +
    61 
    +    res = cli_integration.run(project=project, args=['build', element_name])
    62 
    +    res.assert_success()
    63 
    +
    64 
    +    res = cli_integration.run(project=project, args=[
    65 
    +        'shell', '--build', '--use-buildtree', 'if-available', element_name, '--', 'cat', 'test'
    31 66 
         ])
    32 67 
         res.assert_success()
    68 
    +    assert 'Hi' in res.output
    69 
    +
    70 
    +
    71 
    +@pytest.mark.datafiles(DATA_DIR)
    72 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    73 
    +def test_buildtree_staged_forced_false(cli_integration, tmpdir, datafiles):
    74 
    +    # Test that if we ask not to have a build tree it is not there
    75 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    76 
    +    element_name = 'build-shell/buildtree.bst'
    77 
    +
    78 
    +    res = cli_integration.run(project=project, args=['build', element_name])
    79 
    +    res.assert_success()
    80 
    +
    81 
    +    res = cli_integration.run(project=project, args=[
    82 
    +        'shell', '--build', '--use-buildtree', 'never', element_name, '--', 'cat', 'test'
    83 
    +    ])
    84 
    +    res.assert_shell_error()
    85 
    +
    86 
    +    assert 'Hi' not in res.output
    33 87
    34 88
    35 89 
     @pytest.mark.datafiles(DATA_DIR)
    36 90 
     @pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    37 91 
     def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
    38 
    -    # i.e. test that on a build failure, we can still shell into it
    92 
    +    # Test that we can use a build tree after a failure
    39 93 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    40 94 
         element_name = 'build-shell/buildtree-fail.bst'
    41 95
    ... ... @@ -44,9 +98,10 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
    44 98
    45 99 
         # Assert that file has expected contents
    46 100 
         res = cli_integration.run(project=project, args=[
    47 
    -        'shell', '--build', element_name, '--', 'cat', 'test'
    101 
    +        'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    48 102 
         ])
    49 103 
         res.assert_success()
    104 
    +    assert "Warning: using a buildtree from a failed build" in res.output
    50 105 
         assert 'Hi' in res.output
    51 106
    52 107
    ... ... @@ -80,6 +135,65 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
    80 135
    81 136 
             # Check it's using the cached build tree
    82 137 
             res = cli.run(project=project, args=[
    83 
    -            'shell', '--build', element_name, '--', 'grep', '-q', 'Hi', 'test'
    138 
    +            'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    84 139 
             ])
    85 140 
             res.assert_success()
    141 
    +
    142 
    +
    143 
    +# This test checks for correct behaviour if a buildtree is not present.
    144 
    +@pytest.mark.datafiles(DATA_DIR)
    145 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    146 
    +def test_buildtree_options(cli, tmpdir, datafiles):
    147 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    148 
    +    element_name = 'build-shell/buildtree.bst'
    149 
    +
    150 
    +    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
    151 
    +        # Build the element to push it to cache
    152 
    +        cli.configure({
    153 
    +            'artifacts': {'url': share.repo, 'push': True}
    154 
    +        })
    155 
    +        result = cli.run(project=project, args=['build', element_name])
    156 
    +        result.assert_success()
    157 
    +        assert cli.get_element_state(project, element_name) == 'cached'
    158 
    +
    159 
    +        # Discard the cache
    160 
    +        cli.configure({
    161 
    +            'artifacts': {'url': share.repo, 'push': True},
    162 
    +            'artifactdir': os.path.join(cli.directory, 'artifacts2')
    163 
    +        })
    164 
    +        assert cli.get_element_state(project, element_name) != 'cached'
    165 
    +
    166 
    +        # Pull from cache, but do not include buildtrees.
    167 
    +        result = cli.run(project=project, args=['pull', '--deps', 'all', element_name])
    168 
    +        result.assert_success()
    169 
    +
    170 
    +        # The above is the simplest way I know to create a local cache without any buildtrees.
    171 
    +
    172 
    +        # Check it's not using the cached build tree
    173 
    +        res = cli.run(project=project, args=[
    174 
    +            'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test'
    175 
    +        ])
    176 
    +        res.assert_shell_error()
    177 
    +        assert 'Hi' not in res.output
    178 
    +
    179 
    +        # Check it's not correctly handling the lack of buildtree
    180 
    +        res = cli.run(project=project, args=[
    181 
    +            'shell', '--build', element_name, '--use-buildtree', 'if-available', '--', 'cat', 'test'
    182 
    +        ])
    183 
    +        res.assert_shell_error()
    184 
    +        assert 'Hi' not in res.output
    185 
    +
    186 
    +        # Check it's not using the cached build tree, default is to ask, and fall back to not
    187 
    +        # for non interactive behavior
    188 
    +        res = cli.run(project=project, args=[
    189 
    +            'shell', '--build', element_name, '--', 'cat', 'test'
    190 
    +        ])
    191 
    +        res.assert_shell_error()
    192 
    +        assert 'Hi' not in res.output
    193 
    +
    194 
    +        # Check it's using the cached build tree
    195 
    +        res = cli.run(project=project, args=[
    196 
    +            'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    197 
    +        ])
    198 
    +        res.assert_main_error(ErrorDomain.PROG_NOT_FOUND, None)
    199 
    +        assert 'Hi' not in res.output

  • tests/testutils/runcli.py
    ... ... @@ -153,6 +153,20 @@ class Result():
    153 153 
             assert self.task_error_domain == error_domain, fail_message
    154 154 
             assert self.task_error_reason == error_reason, fail_message
    155 155
    156 
    +    # assert_shell_error()
    157 
    +    #
    158 
    +    # Asserts that the buildstream created a shell and that the task in the
    159 
    +    # shell failed.
    160 
    +    #
    161 
    +    # Args:
    162 
    +    #    fail_message (str): An optional message to override the automatic
    163 
    +    #                        assertion error messages
    164 
    +    # Raises:
    165 
    +    #    (AssertionError): If any of the assertions fail
    166 
    +    #
    167 
    +    def assert_shell_error(self, fail_message=''):
    168 
    +        assert self.exit_code == 1, fail_message
    169 
    +
    156 170 
         # get_tracked_elements()
    157 171 
         #
    158 172 
         # Produces a list of element names on which tracking occurred

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]