GNOME.org
 

[Notes] [Git][BuildStream/buildstream][richardmaw/shell-multi-stage] 8 commits: scriptelement: Split stage into granular methods

Title: GitLab

richardmaw-codethink pushed to branch richardmaw/shell-multi-stage at BuildStream / buildstream

Commits:

24 changed files:

Changes:

  • NEWS
    1 
    +===============
    2 
    +buildstream 1.4
    3 
    +===============
    4 
    +
    5 
    +  o `bst shell` learned the `-e` option for staging multiple elements
    6 
    +    provided the element's kind implements `BST_GRANULAR_STAGE`.
    7 
    +
    1 8 
     =================
    2 9 
     buildstream 1.3.1
    3 10 
     =================

  • buildstream/_frontend/cli.py
    ... ... @@ -572,11 +572,13 @@ def show(app, elements, deps, except_, order, format_):
    572 572 
                   help="Mount a file or directory into the sandbox")
    573 573 
     @click.option('--isolate', is_flag=True, default=False,
    574 574 
                   help='Create an isolated build sandbox')
    575 
    +@click.option('--element', '-e', 'elements', multiple=True,
    576 
    +              type=click.Path(readable=False), required=False)
    575 577 
     @click.argument('element',
    576 
    -                type=click.Path(readable=False))
    578 
    +                type=click.Path(readable=False), required=False)
    577 579 
     @click.argument('command', type=click.STRING, nargs=-1)
    578 580 
     @click.pass_obj
    579 
    -def shell(app, element, sysroot, mount, isolate, build_, command):
    581 
    +def shell(app, element, elements, sysroot, mount, isolate, build_, command):
    580 582 
         """Run a command in the target element's sandbox environment
    581 583
    582 584 
         This will stage a temporary sysroot for running the target
    ... ... @@ -597,13 +599,21 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
    597 599 
         from .._project import HostMount
    598 600 
         from .._pipeline import PipelineSelection
    599 601
    602 
    +    if elements and element is not None:
    603 
    +        command = [element] + list(command)
    604 
    +        element = None
    605 
    +    if not elements and element is not None:
    606 
    +        elements = [element]
    607 
    +    if not elements:
    608 
    +        raise AppError('No elemenets specified to open a shell in')
    609 
    +
    600 610 
         if build_:
    601 611 
             scope = Scope.BUILD
    602 612 
         else:
    603 613 
             scope = Scope.RUN
    604 614
    605 615 
         with app.initialized():
    606 
    -        dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
    616 
    +        dependencies = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
    607 617 
             element = dependencies[0]
    608 618 
             prompt = app.shell_prompt(element)
    609 619 
             mounts = [
    ... ... @@ -611,7 +621,7 @@ def shell(app, element, sysroot, mount, isolate, build_, command):
    611 621 
                 for host_path, path in mount
    612 622 
             ]
    613 623 
             try:
    614 
    -            exitcode = app.stream.shell(element, scope, prompt,
    624 
    +            exitcode = app.stream.shell(dependencies, scope, prompt,
    615 625 
                                             directory=sysroot,
    616 626 
                                             mounts=mounts,
    617 627 
                                             isolate=isolate,

  • buildstream/_stream.py
    ... ... @@ -25,15 +25,17 @@ import stat
    25 25 
     import shlex
    26 26 
     import shutil
    27 27 
     import tarfile
    28 
    -from contextlib import contextmanager
    28 
    +from contextlib import contextmanager, ExitStack
    29 29 
     from tempfile import TemporaryDirectory
    30 30
    31 31 
     from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
    32 32 
     from ._message import Message, MessageType
    33 
    -from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
    34 33 
     from ._pipeline import Pipeline, PipelineSelection
    34 
    +from ._platform import Platform
    35 
    +from .sandbox._config import SandboxConfig
    36 
    +from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
    35 37 
     from . import utils, _yaml, _site
    36 
    -from . import Scope, Consistency
    38 
    +from . import SandboxFlags, Scope, Consistency
    37 39
    38 40
    39 41 
     # Stream()
    ... ... @@ -117,7 +119,7 @@ class Stream():
    117 119 
         # Run a shell
    118 120 
         #
    119 121 
         # Args:
    120 
    -    #    element (Element): An Element object to run the shell for
    122 
    +    #    elements (List of Element): Elements to run the shell for
    121 123 
         #    scope (Scope): The scope for the shell (Scope.BUILD or Scope.RUN)
    122 124 
         #    prompt (str): The prompt to display in the shell
    123 125 
         #    directory (str): A directory where an existing prestaged sysroot is expected, or None
    ... ... @@ -128,7 +130,7 @@ class Stream():
    128 130 
         # Returns:
    129 131 
         #    (int): The exit code of the launched shell
    130 132 
         #
    131 
    -    def shell(self, element, scope, prompt, *,
    133 
    +    def shell(self, elements, scope, prompt, *,
    132 134 
                   directory=None,
    133 135 
                   mounts=None,
    134 136 
                   isolate=False,
    ... ... @@ -140,14 +142,103 @@ class Stream():
    140 142 
             if directory is None:
    141 143 
                 missing_deps = [
    142 144 
                     dep._get_full_name()
    143 
    -                for dep in self._pipeline.dependencies([element], scope)
    145 
    +                for dep in self._pipeline.dependencies(elements, scope)
    144 146 
                     if not dep._cached()
    145 147 
                 ]
    146 148 
                 if missing_deps:
    147 149 
                     raise StreamError("Elements need to be built or downloaded before staging a shell environment",
    148 150 
                                       detail="\n".join(missing_deps))
    149 151
    150 
    -        return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command)
    152 
    +        with ExitStack() as stack:
    153 
    +            # Creation logic duplicated from Element.__sandbox
    154 
    +            # since most of it is creating the tmpdir
    155 
    +            # and deciding whether to make a remote sandbox,
    156 
    +            # which we don't want to.
    157 
    +            if directory is None:
    158 
    +                os.makedirs(self._context.builddir, exist_ok=True)
    159 
    +                rootdir = stack.enter_context(TemporaryDirectory(dir=self._context.builddir))
    160 
    +            else:
    161 
    +                rootdir = directory
    162 
    +
    163 
    +            # SandboxConfig comes from project, element defaults and MetaElement sandbox config
    164 
    +            # In the absence of it being exposed to other APIs and a merging strategy
    165 
    +            # just make it from the project sandbox config.
    166 
    +            sandbox_config = SandboxConfig(_yaml.node_get(self._project._sandbox, int, 'build-uid'),
    167 
    +                                           _yaml.node_get(self._project._sandbox, int, 'build-gid'))
    168 
    +            platform = Platform.get_platform()
    169 
    +            sandbox = platform.create_sandbox(context=self._context,
    170 
    +                                              project=self._project,
    171 
    +                                              directory=rootdir,
    172 
    +                                              stdout=None, stderr=None, config=sandbox_config,
    173 
    +                                              allow_real_directory=not all(e.BST_VIRTUAL_DIRECTORY
    174 
    +                                                                           for e in elements))
    175 
    +
    176 
    +            # Configure the sandbox with the last element taking precedence for config.
    177 
    +            for e in elements:
    178 
    +                e.configure_sandbox(sandbox)
    179 
    +
    180 
    +            # Stage contents if not passed --sysroot
    181 
    +            if not directory:
    182 
    +                if not all(e.BST_GRANULAR_STAGE for e in elements):
    183 
    +                    if len(elements) > 1:
    184 
    +                        raise StreamError(
    185 
    +                            "Elements do not support multiple-element staging",
    186 
    +                            detail=("Elements {} do not support multi-element staging " +
    187 
    +                                    " because element kinds {} do not support BST_GRANULAR_STAGE").format(
    188 
    +                                        ', '.join(e.name for e in elements if not e.BST_GRANULAR_STAGE),
    189 
    +                                        ', '.join(set(e.get_kind() for e in elements))))
    190 
    +                    elements[0].stage(sandbox)
    191 
    +                else:
    192 
    +                    visited = {}
    193 
    +                    for e in elements:
    194 
    +                        e.stage_dependency_artifacts(sandbox, scope, visited=visited)
    195 
    +
    196 
    +                    visited = {}
    197 
    +                    for e in elements:
    198 
    +                        e.integrate_dependencies(sandbox, scope, visited=visited)
    199 
    +
    200 
    +                    for e in elements:
    201 
    +                        e.post_integration_staging(sandbox)
    202 
    +
    203 
    +            environment = {}
    204 
    +            for e in elements:
    205 
    +                environment.update(e.get_environment())
    206 
    +            flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
    207 
    +            shell_command, shell_environment, shell_host_files = self._project.get_shell_config()
    208 
    +            environment['PS1'] = prompt
    209 
    +            # Special configurations for non-isolated sandboxes
    210 
    +            if not isolate:
    211 
    +
    212 
    +                # Open the network, and reuse calling uid/gid
    213 
    +                #
    214 
    +                flags |= SandboxFlags.NETWORK_ENABLED | SandboxFlags.INHERIT_UID
    215 
    +
    216 
    +                # Apply project defined environment vars to set for a shell
    217 
    +                for key, value in _yaml.node_items(shell_environment):
    218 
    +                    environment[key] = value
    219 
    +
    220 
    +                # Setup any requested bind mounts
    221 
    +                if mounts is None:
    222 
    +                    mounts = []
    223 
    +
    224 
    +                for mount in shell_host_files + mounts:
    225 
    +                    if not os.path.exists(mount.host_path):
    226 
    +                        if not mount.optional:
    227 
    +                            self._message(MessageType.WARN,
    228 
    +                                          "Not mounting non-existing host file: {}".format(mount.host_path))
    229 
    +                    else:
    230 
    +                        sandbox.mark_directory(mount.path)
    231 
    +                        sandbox._set_mount_source(mount.path, mount.host_path)
    232 
    +
    233 
    +            if command:
    234 
    +                argv = [arg for arg in command]
    235 
    +            else:
    236 
    +                argv = shell_command
    237 
    +
    238 
    +            self._message(MessageType.STATUS, "Running command", detail=" ".join(argv))
    239 
    +
    240 
    +            # Run shells with network enabled and readonly root.
    241 
    +            return sandbox.run(argv, flags, env=environment)
    151 242
    152 243 
         # build()
    153 244 
         #

  • buildstream/buildelement.py
    ... ... @@ -208,7 +208,6 @@ class BuildElement(Element):
    208 208 
             sandbox.set_environment(self.get_environment())
    209 209
    210 210 
         def stage(self, sandbox):
    211 
    -
    212 211 
             # Stage deps in the sandbox root
    213 212 
             with self.timed_activity("Staging dependencies", silent_nested=True):
    214 213 
                 self.stage_dependency_artifacts(sandbox, Scope.BUILD)
    ... ... @@ -216,9 +215,11 @@ class BuildElement(Element):
    216 215 
             # Run any integration commands provided by the dependencies
    217 216 
             # once they are all staged and ready
    218 217 
             with self.timed_activity("Integrating sandbox"):
    219 
    -            for dep in self.dependencies(Scope.BUILD):
    220 
    -                dep.integrate(sandbox)
    218 
    +            self.integrate_dependencies(sandbox, Scope.BUILD)
    219 
    +
    220 
    +        self.post_integration_staging(sandbox)
    221 221
    222 
    +    def post_integration_staging(self, sandbox):
    222 223 
             # Stage sources in the build root
    223 224 
             self.stage_sources(sandbox, self.get_variable('build-root'))
    224 225

  • buildstream/element.py
    ... ... @@ -75,7 +75,6 @@ Class Reference
    75 75 
     import os
    76 76 
     import re
    77 77 
     import stat
    78 
    -import copy
    79 78 
     from collections import OrderedDict
    80 79 
     from collections.abc import Mapping
    81 80 
     from contextlib import contextmanager
    ... ... @@ -88,7 +87,6 @@ from ._versions import BST_CORE_ARTIFACT_VERSION
    88 87 
     from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain
    89 88 
     from .utils import UtilError
    90 89 
     from . import Plugin, Consistency, Scope
    91 
    -from . import SandboxFlags
    92 90 
     from . import utils
    93 91 
     from . import _cachekey
    94 92 
     from . import _signals
    ... ... @@ -1872,71 +1870,6 @@ class Element(Plugin):
    1872 1870 
             # Notify successful upload
    1873 1871 
             return True
    1874 1872
    1875 
    -    # _shell():
    1876 
    -    #
    1877 
    -    # Connects the terminal with a shell running in a staged
    1878 
    -    # environment
    1879 
    -    #
    1880 
    -    # Args:
    1881 
    -    #    scope (Scope): Either BUILD or RUN scopes are valid, or None
    1882 
    -    #    directory (str): A directory to an existing sandbox, or None
    1883 
    -    #    mounts (list): A list of (str, str) tuples, representing host/target paths to mount
    1884 
    -    #    isolate (bool): Whether to isolate the environment like we do in builds
    1885 
    -    #    prompt (str): A suitable prompt string for PS1
    1886 
    -    #    command (list): An argv to launch in the sandbox
    1887 
    -    #
    1888 
    -    # Returns: Exit code
    1889 
    -    #
    1890 
    -    # If directory is not specified, one will be staged using scope
    1891 
    -    def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None):
    1892 
    -
    1893 
    -        with self._prepare_sandbox(scope, directory) as sandbox:
    1894 
    -            environment = self.get_environment()
    1895 
    -            environment = copy.copy(environment)
    1896 
    -            flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY
    1897 
    -
    1898 
    -            # Fetch the main toplevel project, in case this is a junctioned
    1899 
    -            # subproject, we want to use the rules defined by the main one.
    1900 
    -            context = self._get_context()
    1901 
    -            project = context.get_toplevel_project()
    1902 
    -            shell_command, shell_environment, shell_host_files = project.get_shell_config()
    1903 
    -
    1904 
    -            if prompt is not None:
    1905 
    -                environment['PS1'] = prompt
    1906 
    -
    1907 
    -            # Special configurations for non-isolated sandboxes
    1908 
    -            if not isolate:
    1909 
    -
    1910 
    -                # Open the network, and reuse calling uid/gid
    1911 
    -                #
    1912 
    -                flags |= SandboxFlags.NETWORK_ENABLED | SandboxFlags.INHERIT_UID
    1913 
    -
    1914 
    -                # Apply project defined environment vars to set for a shell
    1915 
    -                for key, value in _yaml.node_items(shell_environment):
    1916 
    -                    environment[key] = value
    1917 
    -
    1918 
    -                # Setup any requested bind mounts
    1919 
    -                if mounts is None:
    1920 
    -                    mounts = []
    1921 
    -
    1922 
    -                for mount in shell_host_files + mounts:
    1923 
    -                    if not os.path.exists(mount.host_path):
    1924 
    -                        if not mount.optional:
    1925 
    -                            self.warn("Not mounting non-existing host file: {}".format(mount.host_path))
    1926 
    -                    else:
    1927 
    -                        sandbox.mark_directory(mount.path)
    1928 
    -                        sandbox._set_mount_source(mount.path, mount.host_path)
    1929 
    -
    1930 
    -            if command:
    1931 
    -                argv = [arg for arg in command]
    1932 
    -            else:
    1933 
    -                argv = shell_command
    1934 
    -
    1935 
    -            self.status("Running command", detail=" ".join(argv))
    1936 
    -
    1937 
    -            # Run shells with network enabled and readonly root.
    1938 
    -            return sandbox.run(argv, flags, env=environment)
    1939 
    -
    1940 1873 
         # _open_workspace():
    1941 1874 
         #
    1942 1875 
         # "Open" a workspace for this element

  • buildstream/plugins/elements/autotools.py
    ... ... @@ -62,6 +62,8 @@ from buildstream import BuildElement
    62 62 
     class AutotoolsElement(BuildElement):
    63 63 
         # Supports virtual directories (required for remote execution)
    64 64 
         BST_VIRTUAL_DIRECTORY = True
    65 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    66 
    +    BST_GRANULAR_STAGE = True
    65 67
    66 68
    67 69 
     # Plugin entry point

  • buildstream/plugins/elements/cmake.py
    ... ... @@ -61,6 +61,8 @@ from buildstream import BuildElement
    61 61 
     class CMakeElement(BuildElement):
    62 62 
         # Supports virtual directories (required for remote execution)
    63 63 
         BST_VIRTUAL_DIRECTORY = True
    64 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    65 
    +    BST_GRANULAR_STAGE = True
    64 66
    65 67
    66 68 
     # Plugin entry point

  • buildstream/plugins/elements/compose.py
    ... ... @@ -58,6 +58,9 @@ class ComposeElement(Element):
    58 58 
         # This plugin has been modified to avoid the use of Sandbox.get_directory
    59 59 
         BST_VIRTUAL_DIRECTORY = True
    60 60
    61 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    62 
    +    BST_GRANULAR_STAGE = True
    63 
    +
    61 64 
         def configure(self, node):
    62 65 
             self.node_validate(node, [
    63 66 
                 'integrate', 'include', 'exclude', 'include-orphans'

  • buildstream/plugins/elements/distutils.py
    ... ... @@ -36,7 +36,8 @@ from buildstream import BuildElement
    36 36
    37 37 
     # Element implementation for the python 'distutils' kind.
    38 38 
     class DistutilsElement(BuildElement):
    39 
    -    pass
    39 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    40 
    +    BST_GRANULAR_STAGE = True
    40 41
    41 42
    42 43 
     # Plugin entry point

  • buildstream/plugins/elements/import.py
    ... ... @@ -43,6 +43,8 @@ class ImportElement(BuildElement):
    43 43
    44 44 
         # This plugin has been modified to avoid the use of Sandbox.get_directory
    45 45 
         BST_VIRTUAL_DIRECTORY = True
    46 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    47 
    +    BST_GRANULAR_STAGE = True
    46 48
    47 49 
         def configure(self, node):
    48 50 
             self.source = self.node_subst_member(node, 'source')
    ... ... @@ -67,6 +69,9 @@ class ImportElement(BuildElement):
    67 69 
         def stage(self, sandbox):
    68 70 
             pass
    69 71
    72 
    +    def post_integration_staging(self, sandbox):
    73 
    +        pass
    74 
    +
    70 75 
         def assemble(self, sandbox):
    71 76
    72 77 
             # Stage sources into the input directory

  • buildstream/plugins/elements/make.py
    ... ... @@ -43,6 +43,8 @@ from buildstream import BuildElement
    43 43 
     class MakeElement(BuildElement):
    44 44 
         # Supports virtual directories (required for remote execution)
    45 45 
         BST_VIRTUAL_DIRECTORY = True
    46 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    47 
    +    BST_GRANULAR_STAGE = True
    46 48
    47 49
    48 50 
     # Plugin entry point

  • buildstream/plugins/elements/makemaker.py
    ... ... @@ -36,7 +36,8 @@ from buildstream import BuildElement
    36 36
    37 37 
     # Element implementation for the 'makemaker' kind.
    38 38 
     class MakeMakerElement(BuildElement):
    39 
    -    pass
    39 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    40 
    +    BST_GRANULAR_STAGE = True
    40 41
    41 42
    42 43 
     # Plugin entry point

  • buildstream/plugins/elements/manual.py
    ... ... @@ -36,7 +36,8 @@ from buildstream import BuildElement
    36 36
    37 37 
     # Element implementation for the 'manual' kind.
    38 38 
     class ManualElement(BuildElement):
    39 
    -    pass
    39 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    40 
    +    BST_GRANULAR_STAGE = True
    40 41
    41 42
    42 43 
     # Plugin entry point

  • buildstream/plugins/elements/meson.py
    ... ... @@ -58,6 +58,8 @@ from buildstream import BuildElement
    58 58 
     class MesonElement(BuildElement):
    59 59 
         # Supports virtual directories (required for remote execution)
    60 60 
         BST_VIRTUAL_DIRECTORY = True
    61 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    62 
    +    BST_GRANULAR_STAGE = True
    61 63
    62 64
    63 65 
     # Plugin entry point

  • buildstream/plugins/elements/modulebuild.py
    ... ... @@ -36,7 +36,8 @@ from buildstream import BuildElement
    36 36
    37 37 
     # Element implementation for the 'modulebuild' kind.
    38 38 
     class ModuleBuildElement(BuildElement):
    39 
    -    pass
    39 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    40 
    +    BST_GRANULAR_STAGE = True
    40 41
    41 42
    42 43 
     # Plugin entry point

  • buildstream/plugins/elements/pip.py
    ... ... @@ -36,7 +36,8 @@ from buildstream import BuildElement
    36 36
    37 37 
     # Element implementation for the 'pip' kind.
    38 38 
     class PipElement(BuildElement):
    39 
    -    pass
    39 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    40 
    +    BST_GRANULAR_STAGE = True
    40 41
    41 42
    42 43 
     # Plugin entry point

  • buildstream/plugins/elements/qmake.py
    ... ... @@ -38,6 +38,8 @@ from buildstream import BuildElement
    38 38 
     class QMakeElement(BuildElement):
    39 39 
         # Supports virtual directories (required for remote execution)
    40 40 
         BST_VIRTUAL_DIRECTORY = True
    41 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    42 
    +    BST_GRANULAR_STAGE = True
    41 43
    42 44
    43 45 
     # Plugin entry point

  • buildstream/plugins/elements/script.py
    ... ... @@ -42,6 +42,9 @@ import buildstream
    42 42 
     class ScriptElement(buildstream.ScriptElement):
    43 43 
         # pylint: disable=attribute-defined-outside-init
    44 44
    45 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    46 
    +    BST_GRANULAR_STAGE = True
    47 
    +
    45 48 
         def configure(self, node):
    46 49 
             for n in self.node_get_member(node, list, 'layout', []):
    47 50 
                 dst = self.node_subst_member(n, 'destination')

  • buildstream/plugins/elements/stack.py
    ... ... @@ -33,6 +33,9 @@ class StackElement(Element):
    33 33 
         # This plugin has been modified to avoid the use of Sandbox.get_directory
    34 34 
         BST_VIRTUAL_DIRECTORY = True
    35 35
    36 
    +    # This plugin has been modified to permit splitting stage, integration and post-stage
    37 
    +    BST_GRANULAR_STAGE = True
    38 
    +
    36 39 
         def configure(self, node):
    37 40 
             pass
    38 41

  • buildstream/scriptelement.py
    ... ... @@ -215,17 +215,32 @@ class ScriptElement(Element):
    215 215 
         def stage(self, sandbox):
    216 216
    217 217 
             # Stage the elements, and run integration commands where appropriate.
    218 
    -        if not self.__layout:
    219 
    -            # if no layout set, stage all dependencies into /
    220 
    -            for build_dep in self.dependencies(Scope.BUILD, recurse=False):
    221 
    -                with self.timed_activity("Staging {} at /"
    222 
    -                                         .format(build_dep.name), silent_nested=True):
    223 
    -                    build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/")
    218 
    +        self.stage_dependency_artifacts(sandbox, Scope.BUILD)
    219 
    +        self.integrate_dependencies(sandbox, Scope.BUILD)
    220 
    +
    221 
    +        self.post_integration_staging(sandbox)
    222 
    +
    223 
    +    def stage_dependency_artifacts(self, sandbox, scope, *, path=None,
    224 
    +                                   include=None, exclude=None, orphans=True, visited=None):
    225 
    +        if scope is not Scope.BUILD:
    226 
    +            super().stage_dependency_artifacts(sandbox, scope, path=path,
    227 
    +                                               include=include,
    228 
    +                                               exclude=exclude,
    229 
    +                                               orphans=orphans,
    230 
    +                                               visited=visited)
    231 
    +            return
    232 
    +
    233 
    +        if path is None:
    234 
    +            path = "/"
    235 
    +        if visited is None:
    236 
    +            visited = {}
    224 237
    238 
    +        if not self.__layout:
    239 
    +            # if no layout set, stage all dependencies into path
    225 240 
                 for build_dep in self.dependencies(Scope.BUILD, recurse=False):
    226 
    -                with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
    227 
    -                    for dep in build_dep.dependencies(Scope.RUN):
    228 
    -                        dep.integrate(sandbox)
    241 
    +                with self.timed_activity("Staging {} at {}"
    242 
    +                                         .format(build_dep.name, path), silent_nested=True):
    243 
    +                    build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path=path, visited=visited)
    229 244 
             else:
    230 245 
                 # If layout, follow its rules.
    231 246 
                 for item in self.__layout:
    ... ... @@ -236,17 +251,33 @@ class ScriptElement(Element):
    236 251
    237 252 
                     element = self.search(Scope.BUILD, item['element'])
    238 253 
                     if item['destination'] == '/':
    239 
    -                    with self.timed_activity("Staging {} at /".format(element.name),
    254 
    +                    with self.timed_activity("Staging {} at {}".format(element.name, path),
    240 255 
                                                  silent_nested=True):
    241 
    -                        element.stage_dependency_artifacts(sandbox, Scope.RUN)
    256 
    +                        element.stage_dependency_artifacts(sandbox, Scope.RUN, path=path, visited=visited)
    242 257 
                     else:
    258 
    +                    subpath = os.path.join(path, item['destination'])
    243 259 
                         with self.timed_activity("Staging {} at {}"
    244 
    -                                             .format(element.name, item['destination']),
    260 
    +                                             .format(element.name, subpath),
    245 261 
                                                  silent_nested=True):
    246 262 
                             virtual_dstdir = sandbox.get_virtual_directory()
    247 
    -                        virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True)
    248 
    -                        element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
    263 
    +                        virtual_dstdir.descend(subpath.lstrip(os.sep).split(os.sep), create=True)
    264 
    +                        element.stage_dependency_artifacts(sandbox, Scope.RUN, path=subpath, visited=visited)
    265 
    +
    266 
    +    def integrate_dependencies(self, sandbox, scope, *, visited=None):
    267 
    +        if scope is not Scope.BUILD:
    268 
    +            super().integrate_dependencies(sandbox, scope, visited=visited)
    269 
    +            return
    270 
    +
    271 
    +        if visited is None:
    272 
    +            visited = {}
    249 273
    274 
    +        if not self.__layout:
    275 
    +            for build_dep in self.dependencies(Scope.BUILD, recurse=False):
    276 
    +                with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
    277 
    +                    for dep in build_dep.dependencies(Scope.RUN, visited=visited):
    278 
    +                        dep.integrate(sandbox)
    279 
    +        else:
    280 
    +            # If layout, follow its rules.
    250 281 
                 for item in self.__layout:
    251 282
    252 283 
                     # Skip layout members which dont stage an element
    ... ... @@ -259,9 +290,10 @@ class ScriptElement(Element):
    259 290 
                     if item['destination'] == '/':
    260 291 
                         with self.timed_activity("Integrating {}".format(element.name),
    261 292 
                                                  silent_nested=True):
    262 
    -                        for dep in element.dependencies(Scope.RUN):
    293 
    +                        for dep in element.dependencies(Scope.RUN, visited=visited):
    263 294 
                                 dep.integrate(sandbox)
    264 295
    296 
    +    def post_integration_staging(self, sandbox):
    265 297 
             install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep)
    266 298 
             sandbox.get_virtual_directory().descend(install_root_path_components, create=True)
    267 299

  • tests/integration/project/elements/shell/adds-bar.bst
    1 
    +kind: manual
    2 
    +depends:
    3 
    +- base.bst
    4 
    +
    5 
    +config:
    6 
    +  install-commands:
    7 
    +  - |
    8 
    +    install -D -m775 /proc/self/fd/0 %{install-root}%{bindir}/bar <<\EOF
    9 
    +    #!/bin/sh
    10 
    +    echo bar
    11 
    +    EOF

  • tests/integration/project/elements/shell/adds-foo.bst
    1 
    +kind: manual
    2 
    +depends:
    3 
    +- base.bst
    4 
    +
    5 
    +config:
    6 
    +  install-commands:
    7 
    +  - |
    8 
    +    install -D -m775 /proc/self/fd/0 %{install-root}%{bindir}/foo <<\EOF
    9 
    +    #!/bin/sh
    10 
    +    echo foo
    11 
    +    EOF

  • tests/integration/project/elements/integration.bst → tests/integration/project/elements/shell/integration.bst

  • tests/integration/shell.py
    ... ... @@ -29,9 +29,11 @@ DATA_DIR = os.path.join(
    29 29 
     #    element (str): The element to build and run a shell with
    30 30 
     #    isolate (bool): Whether to pass --isolate to `bst shell`
    31 31 
     #
    32 
    -def execute_shell(cli, project, command, *, config=None, mount=None, element='base.bst', isolate=False):
    32 
    +def execute_shell(cli, project, command, *, config=None, mount=None, element='base.bst', elements=None, isolate=False):
    33 33 
         # Ensure the element is built
    34 
    -    result = cli.run(project=project, project_config=config, args=['build', element])
    34 
    +    if elements is None:
    35 
    +        elements = [element]
    36 
    +    result = cli.run(project=project, project_config=config, args=['build'] + elements)
    35 37 
         assert result.exit_code == 0
    36 38
    37 39 
         args = ['shell']
    ... ... @@ -40,7 +42,7 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element='ba
    40 42 
         if mount is not None:
    41 43 
             host_path, target_path = mount
    42 44 
             args += ['--mount', host_path, target_path]
    43 
    -    args += [element, '--'] + command
    45 
    +    args += ["-e" + e for e in elements] + ['--'] + command
    44 46
    45 47 
         return cli.run(project=project, project_config=config, args=args)
    46 48
    ... ... @@ -345,10 +347,51 @@ def test_sysroot_workspace_visible(cli, tmpdir, datafiles):
    345 347
    346 348
    347 349 
     # Test system integration commands can access devices in /dev
    350 
    +@pytest.mark.integration
    348 351 
     @pytest.mark.datafiles(DATA_DIR)
    349 352 
     def test_integration_devices(cli, tmpdir, datafiles):
    350 353 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    351 
    -    element_name = 'integration.bst'
    354 
    +    element_name = 'shell/integration.bst'
    352 355
    353 356 
         result = execute_shell(cli, project, ["true"], element=element_name)
    354 357 
         assert result.exit_code == 0
    358 
    +
    359 
    +
    360 
    +# Test multiple element shell
    361 
    +@pytest.mark.integration
    362 
    +@pytest.mark.datafiles(DATA_DIR)
    363 
    +def test_shell_multiple_elements(cli, tmpdir, datafiles):
    364 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    365 
    +
    366 
    +    result = execute_shell(cli, project, ["sh", "-c", "foo && bar"],
    367 
    +                           elements=["shell/adds-foo.bst", "shell/adds-bar.bst"])
    368 
    +    assert result.exit_code == 0
    369 
    +
    370 
    +
    371 
    +# Test multiple element build shell
    372 
    +@pytest.mark.integration
    373 
    +@pytest.mark.datafiles(DATA_DIR)
    374 
    +def test_shell_multiple_workspace(cli, tmpdir, datafiles):
    375 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    376 
    +    elements = {'workspace/workspace-mount.bst': os.path.join(cli.directory, 'workspace-mount'),
    377 
    +                'make/makehello.bst': os.path.join(cli.directory, 'makehello')}
    378 
    +
    379 
    +    for element, workspace in elements.items():
    380 
    +        res = cli.run(project=project, args=['workspace', 'open', element, workspace])
    381 
    +        assert res.exit_code == 0
    382 
    +
    383 
    +    for workspace in elements.values():
    384 
    +        with open(os.path.join(workspace, "workspace-exists"), "w") as f:
    385 
    +            pass
    386 
    +
    387 
    +    # Ensure the dependencies of our build failing element are built
    388 
    +    result = cli.run(project=project, args=['build', 'base.bst'])
    389 
    +    assert result.exit_code == 0
    390 
    +
    391 
    +    args = ['shell', '--build'] + ['-e' + e for e in elements]
    392 
    +    args += ['--', 'sh', '-c',
    393 
    +             'test -e /buildstream/test/workspace/workspace-mount.bst/workspace-exists && \
    394 
    +              test -e /buildstream/test/make/makehello.bst/workspace-exists']
    395 
    +    result = cli.run(project=project, args=args)
    396 
    +    assert result.exit_code == 0
    397 
    +    assert result.output == ''

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]