GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/sysroot_dependencies_2] Add support for sysroot'ed dependencies in BuildElement and ScriptElement

Title: GitLab

Valentin David pushed to branch valentindavid/sysroot_dependencies_2 at BuildStream / buildstream

Commits:

29 changed files:

Changes:

  • buildstream/_sysroot_dependency_loader.py
    1 
    +#
    2 
    +#  Copyright (C) 2019 Codethink Limited
    3 
    +#
    4 
    +#  This program is free software; you can redistribute it and/or
    5 
    +#  modify it under the terms of the GNU Lesser General Public
    6 
    +#  License as published by the Free Software Foundation; either
    7 
    +#  version 2 of the License, or (at your option) any later version.
    8 
    +#
    9 
    +#  This library is distributed in the hope that it will be useful,
    10 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    11 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    12 
    +#  Lesser General Public License for more details.
    13 
    +#
    14 
    +#  You should have received a copy of the GNU Lesser General Public
    15 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    16 
    +#
    17 
    +#  Authors:
    18 
    +#        Valentin David <valentin david codethink co uk>
    19 
    +
    20 
    +import os
    21 
    +import itertools
    22 
    +from collections.abc import Mapping
    23 
    +
    24 
    +from .dependency_loader import DependencyLoader, Dependency
    25 
    +from . import Element, ElementError, Scope, SandboxFlags
    26 
    +
    27 
    +
    28 
    +# SysrootDependencyLoader():
    29 
    +#
    30 
    +# `SysrootDependencyLoader` implements a `DependencyLoader` to extract
    31 
    +# sysroot'ed dependencies.
    32 
    +class SysrootDependencyLoader(DependencyLoader):
    33 
    +
    34 
    +    def get_dependencies(self, node):
    35 
    +        sysroots = self.node_get_member(node, list, 'sysroots', default=[])
    36 
    +        dependencies = []
    37 
    +
    38 
    +        for sysroot in sysroots:
    39 
    +            depends = self.node_get_member(sysroot, list, 'depends', default=[])
    40 
    +            build_depends = self.node_get_member(sysroot, list, 'build-depends', default=[])
    41 
    +            depends_iter = itertools.product(['all'], depends)
    42 
    +            build_depends_iter = itertools.product(['build'], build_depends)
    43 
    +            for default_type, dep in itertools.chain(depends_iter, build_depends_iter):
    44 
    +                if isinstance(dep, Mapping):
    45 
    +                    provenance = self.node_provenance(dep)
    46 
    +                    filename = self.node_get_member(node, str, 'filename')
    47 
    +                    dep_type = self.node_get_member(node, str, 'type', default=default_type)
    48 
    +                    junction = self.node_get_member(node, str, 'junction', default=None)
    49 
    +                    dependencies.append(Dependency(filename, dep_type=dep_type,
    50 
    +                                                   junction=junction, provenance=provenance))
    51 
    +                else:
    52 
    +                    provenance = self.node_provenance(sysroot)
    53 
    +                    dependencies.append(Dependency(dep, dep_type=default_type, provenance=provenance))
    54 
    +
    55 
    +        return dependencies
    56 
    +
    57 
    +
    58 
    +# SysrootHelper():
    59 
    +#
    60 
    +# `SysrootHelper` should be used in element plugins that use
    61 
    +# `SysrootDependencyLoader` as dependency loader. It provides
    62 
    +# The implementation for staging.
    63 
    +class SysrootHelper:
    64 
    +
    65 
    +    CONFIG_KEYS = ['sysroots']
    66 
    +    __layout = []
    67 
    +
    68 
    +    def __init__(self, element, node):
    69 
    +
    70 
    +        self.__element = element
    71 
    +
    72 
    +        for sysroot in self.__element.node_get_member(node, list, 'sysroots', []):
    73 
    +            self.__element.node_validate(sysroot, ['path', 'depends', 'build-depends'])
    74 
    +            path = self.__element.node_subst_member(sysroot, 'path')
    75 
    +            depends = self.__element.node_get_member(sysroot, list, 'depends', default=[])
    76 
    +            build_depends = self.__element.node_get_member(sysroot, list, 'build-depends', default=[])
    77 
    +            for dep in itertools.chain(depends, build_depends):
    78 
    +                if isinstance(dep, Mapping):
    79 
    +                    self.__element.node_validate(dep, ['filename', 'type', 'junction'])
    80 
    +                    filename = self.__element.node_get_member(dep, str, 'filename')
    81 
    +                    junction = self.__element.node_get_member(dep, str, 'junction', default=None)
    82 
    +                else:
    83 
    +                    filename = dep
    84 
    +                    junction = None
    85 
    +                self.layout_add(filename, path, junction=junction)
    86 
    +
    87 
    +    # layout_add():
    88 
    +    #
    89 
    +    # Adds a destination where a dependency should be staged
    90 
    +    #
    91 
    +    # Args:
    92 
    +    #    element (str): Element name of the dependency
    93 
    +    #    destination (str): Path where element will be staged
    94 
    +    #    junction (str): Junction of the dependency
    95 
    +    #
    96 
    +    # If `junction` is None, then the dependency should be in the same
    97 
    +    # project as the current element.
    98 
    +    #
    99 
    +    # If `junction` is ignored or `Element.IGNORE_JUNCTION`, the
    100 
    +    # junction of the dependency is not checked.  This is for backward
    101 
    +    # compliancy and should not be used.
    102 
    +    #
    103 
    +    # If `element` is None, the destination will just
    104 
    +    # be marked in the sandbox.
    105 
    +    def layout_add(self, element, destination, *, junction=Element.IGNORE_JUNCTION):
    106 
    +        #
    107 
    +        # Even if this is an empty list by default, make sure that its
    108 
    +        # instance data instead of appending stuff directly onto class data.
    109 
    +        #
    110 
    +        if not self.__layout:
    111 
    +            self.__layout = []
    112 
    +        item = {'element': element,
    113 
    +                'destination': destination}
    114 
    +        if junction is not Element.IGNORE_JUNCTION:
    115 
    +            item['junction'] = junction
    116 
    +        self.__layout.append(item)
    117 
    +
    118 
    +    # validate():
    119 
    +    #
    120 
    +    # Verify that elements in layouts are dependencies.
    121 
    +    #
    122 
    +    # Raises:
    123 
    +    #    (ElementError): When a element is not in the dependencies
    124 
    +    #
    125 
    +    # This method is only useful when SysrootHelper.layout_add
    126 
    +    # has been called directly.
    127 
    +    #
    128 
    +    # This should be called in implementation of Plugin.preflight.
    129 
    +    def validate(self):
    130 
    +        if self.__layout:
    131 
    +            # Cannot proceed if layout specifies an element that isn't part
    132 
    +            # of the dependencies.
    133 
    +            for item in self.__layout:
    134 
    +                if not item['element']:
    135 
    +                    if not self.__search(item):
    136 
    +                        raise ElementError("{}: '{}' in layout not found in dependencies"
    137 
    +                                           .format(self.__element, item['element']))
    138 
    +
    139 
    +    # stage():
    140 
    +    #
    141 
    +    # Stage dependencies and integrate root dependencies
    142 
    +    #
    143 
    +    # Args:
    144 
    +    #    stage_all (bool): Whether to stage all dependencies, not just the ones mapped
    145 
    +    #
    146 
    +    def stage(self, sandbox, stage_all):
    147 
    +
    148 
    +        staged = set()
    149 
    +        sysroots = {}
    150 
    +
    151 
    +        for item in self.__layout:
    152 
    +
    153 
    +            # Skip layout members which dont stage an element
    154 
    +            if not item['element']:
    155 
    +                continue
    156 
    +
    157 
    +            element = self.__search(item)
    158 
    +            staged.add(element)
    159 
    +            if item['destination'] not in sysroots:
    160 
    +                sysroots[item['destination']] = [element]
    161 
    +            else:
    162 
    +                sysroots[item['destination']].append(element)
    163 
    +
    164 
    +        if stage_all or not self.__layout:
    165 
    +            for build_dep in self.__element.dependencies(Scope.BUILD, recurse=False):
    166 
    +                if build_dep in staged:
    167 
    +                    continue
    168 
    +                if '/' not in sysroots:
    169 
    +                    sysroots['/'] = [build_dep]
    170 
    +                else:
    171 
    +                    sysroots['/'].append(build_dep)
    172 
    +
    173 
    +        for sysroot, deps in sysroots.items():
    174 
    +            with self.__element.timed_activity("Staging dependencies at {}".format(sysroot), silent_nested=True):
    175 
    +                if sysroot != '/':
    176 
    +                    virtual_dstdir = sandbox.get_virtual_directory()
    177 
    +                    virtual_dstdir.descend(sysroot.lstrip(os.sep).split(os.sep), create=True)
    178 
    +                all_deps = set()
    179 
    +                for dep in deps:
    180 
    +                    for run_dep in dep.dependencies(Scope.RUN):
    181 
    +                        all_deps.add(run_dep)
    182 
    +                self.__element.stage_dependency_artifacts(sandbox, Scope.BUILD, path=sysroot, dependencies=all_deps)
    183 
    +
    184 
    +        with sandbox.batch(SandboxFlags.NONE):
    185 
    +            for item in self.__layout:
    186 
    +
    187 
    +                # Skip layout members which dont stage an element
    188 
    +                if not item['element']:
    189 
    +                    continue
    190 
    +
    191 
    +                element = self.__search(item)
    192 
    +
    193 
    +                # Integration commands can only be run for elements staged to /
    194 
    +                if item['destination'] == '/':
    195 
    +                    with self.__element.timed_activity("Integrating {}".format(element.name),
    196 
    +                                                       silent_nested=True):
    197 
    +                        for dep in element.dependencies(Scope.RUN):
    198 
    +                            element.integrate(sandbox)
    199 
    +
    200 
    +            if stage_all or not self.__layout:
    201 
    +                for build_dep in self.__element.dependencies(Scope.BUILD, recurse=False):
    202 
    +                    if build_dep in staged:
    203 
    +                        continue
    204 
    +
    205 
    +                    with self.__element.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
    206 
    +                        for dep in build_dep.dependencies(Scope.RUN):
    207 
    +                            dep.integrate(sandbox)
    208 
    +
    209 
    +    # has_sysroots():
    210 
    +    #
    211 
    +    # Tells whether any element has been mapped
    212 
    +    #
    213 
    +    # Returns:
    214 
    +    #    (bool): Whether any element has been mapped
    215 
    +    def has_sysroots(self):
    216 
    +        return bool(self.__layout)
    217 
    +
    218 
    +    # get_unique_key():
    219 
    +    #
    220 
    +    # Returns a value usable for an element unique key
    221 
    +    #
    222 
    +    # Returns:
    223 
    +    #    (dict): A dictionary that uniquely identify the mapping configuration
    224 
    +    def get_unique_key(self):
    225 
    +        return self.__layout
    226 
    +
    227 
    +    # configure_sandbox():
    228 
    +    #
    229 
    +    # Configure the sandbox. Mark required directories in the sandbox.
    230 
    +    #
    231 
    +    # Args:
    232 
    +    #    extra_directories (list(str)): Extra directories to mark
    233 
    +    #
    234 
    +    # Because Sandbox.mark_directory should be called
    235 
    +    # only once, marked directories should passed as `extra_directories`
    236 
    +    # instead of being marked directly.
    237 
    +    def configure_sandbox(self, sandbox, extra_directories):
    238 
    +
    239 
    +        directories = {directory: False for directory in extra_directories}
    240 
    +
    241 
    +        for item in self.__layout:
    242 
    +            destination = item['destination']
    243 
    +            was_artifact = directories.get(destination, False)
    244 
    +            directories[destination] = item['element'] or was_artifact
    245 
    +
    246 
    +        for directory, artifact in directories.items():
    247 
    +            # Root does not need to be marked as it is always mounted
    248 
    +            # with artifact (unless explicitly marked non-artifact)
    249 
    +            if directory != '/':
    250 
    +                sandbox.mark_directory(directory, artifact=artifact)
    251 
    +
    252 
    +    #
    253 
    +    # Private methods
    254 
    +    #
    255 
    +
    256 
    +    def __search(self, item):
    257 
    +        if 'junction' in item:
    258 
    +            return self.__element.search(Scope.BUILD, item['element'], junction=item['junction'])
    259 
    +        else:
    260 
    +            return self.__element.search(Scope.BUILD, item['element'])

  • buildstream/buildelement.py
    ... ... @@ -135,8 +135,10 @@ artifact collection purposes.
    135 135 
     """
    136 136
    137 137 
     import os
    138 
    -from . import Element, Scope
    138 
    +
    139 
    +from . import Element
    139 140 
     from . import SandboxFlags
    141 
    +from ._sysroot_dependency_loader import SysrootDependencyLoader, SysrootHelper
    140 142
    141 143
    142 144 
     # This list is preserved because of an unfortunate situation, we
    ... ... @@ -157,17 +159,20 @@ _command_steps = ['configure-commands',
    157 159
    158 160 
     class BuildElement(Element):
    159 161
    162 
    +    DEPENDENCY_LOADER = SysrootDependencyLoader
    163 
    +
    160 164 
         #############################################################
    161 165 
         #             Abstract Method Implementations               #
    162 166 
         #############################################################
    163 167 
         def configure(self, node):
    164 168
    165 169 
             self.__commands = {}  # pylint: disable=attribute-defined-outside-init
    170 
    +        self.__sysroots = SysrootHelper(self, node)  # pylint: disable=attribute-defined-outside-init
    166 171
    167 172 
             # FIXME: Currently this forcefully validates configurations
    168 173 
             #        for all BuildElement subclasses so they are unable to
    169 174 
             #        extend the configuration
    170 
    -        self.node_validate(node, _command_steps)
    175 
    +        self.node_validate(node, _command_steps + SysrootHelper.CONFIG_KEYS)
    171 176
    172 177 
             for command_name in _legacy_command_steps:
    173 178 
                 if command_name in _command_steps:
    ... ... @@ -191,6 +196,9 @@ class BuildElement(Element):
    191 196 
             if self.get_variable('notparallel'):
    192 197 
                 dictionary['notparallel'] = True
    193 198
    199 
    +        if self.__sysroots.has_sysroots():
    200 
    +            dictionary['sysroots'] = self.__sysroots.get_unique_key()
    201 
    +
    194 202 
             return dictionary
    195 203
    196 204 
         def configure_sandbox(self, sandbox):
    ... ... @@ -198,8 +206,8 @@ class BuildElement(Element):
    198 206 
             install_root = self.get_variable('install-root')
    199 207
    200 208 
             # Tell the sandbox to mount the build root and install root
    201 
    -        sandbox.mark_directory(build_root)
    202 
    -        sandbox.mark_directory(install_root)
    209 
    +        self.__sysroots.configure_sandbox(sandbox, [build_root,
    210 
    +                                                    install_root])
    203 211
    204 212 
             # Allow running all commands in a specified subdirectory
    205 213 
             command_subdir = self.get_variable('command-subdir')
    ... ... @@ -217,15 +225,7 @@ class BuildElement(Element):
    217 225
    218 226 
         def stage(self, sandbox):
    219 227
    220 
    -        # Stage deps in the sandbox root
    221 
    -        with self.timed_activity("Staging dependencies", silent_nested=True):
    222 
    -            self.stage_dependency_artifacts(sandbox, Scope.BUILD)
    223 
    -
    224 
    -        # Run any integration commands provided by the dependencies
    225 
    -        # once they are all staged and ready
    226 
    -        with sandbox.batch(SandboxFlags.NONE, label="Integrating sandbox"):
    227 
    -            for dep in self.dependencies(Scope.BUILD):
    228 
    -                dep.integrate(sandbox)
    228 
    +        self.__sysroots.stage(sandbox, True)
    229 229
    230 230 
             # Stage sources in the build root
    231 231 
             self.stage_sources(sandbox, self.get_variable('build-root'))

  • buildstream/element.py
    ... ... @@ -677,7 +677,8 @@ class Element(Plugin):
    677 677 
             return link_result.combine(copy_result)
    678 678
    679 679 
         def stage_dependency_artifacts(self, sandbox, scope, *, path=None,
    680 
    -                                   include=None, exclude=None, orphans=True):
    680 
    +                                   include=None, exclude=None, orphans=True,
    681 
    +                                   dependencies=None):
    681 682 
             """Stage element dependencies in scope
    682 683
    683 684 
             This is primarily a convenience wrapper around
    ... ... @@ -692,6 +693,7 @@ class Element(Plugin):
    692 693 
                include (list): An optional list of domains to include files from
    693 694 
                exclude (list): An optional list of domains to exclude files from
    694 695 
                orphans (bool): Whether to include files not spoken for by split domains
    696 
    +           dependencies (list): An optional list of dependencies to stage
    695 697
    696 698 
             Raises:
    697 699 
                (:class:`.ElementError`): If any of the dependencies in `scope` have not
    ... ... @@ -707,7 +709,9 @@ class Element(Plugin):
    707 709 
             if self.__can_build_incrementally() and workspace.last_successful:
    708 710 
                 old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful)
    709 711
    710 
    -        for dep in self.dependencies(scope):
    712 
    +        if dependencies is None:
    713 
    +            dependencies = self.dependencies(scope)
    714 
    +        for dep in dependencies:
    711 715 
                 # If we are workspaced, and we therefore perform an
    712 716 
                 # incremental build, we must ensure that we update the mtimes
    713 717 
                 # of any files created by our dependencies since the last

  • buildstream/plugins/elements/script.py
    ... ... @@ -36,6 +36,7 @@ The default configuration and possible options are as such:
    36 36 
     """
    37 37
    38 38 
     import buildstream
    39 
    +from buildstream._sysroot_dependency_loader import DependencyLoader
    39 40
    40 41
    41 42 
     # Element implementation for the 'script' kind.
    ... ... @@ -46,6 +47,8 @@ class ScriptElement(buildstream.ScriptElement):
    46 47 
         BST_VIRTUAL_DIRECTORY = True
    47 48
    48 49 
         def configure(self, node):
    50 
    +        super().configure(node)
    51 
    +
    49 52 
             for n in self.node_get_member(node, list, 'layout', []):
    50 53 
                 dst = self.node_subst_member(n, 'destination')
    51 54 
                 elm = self.node_subst_member(n, 'element', None)
    ... ... @@ -53,7 +56,7 @@ class ScriptElement(buildstream.ScriptElement):
    53 56
    54 57 
             self.node_validate(node, [
    55 58 
                 'commands', 'root-read-only', 'layout'
    56 
    -        ])
    59 
    +        ] + self.COMMON_CONFIG_KEYS)
    57 60
    58 61 
             cmds = self.node_subst_list(node, "commands")
    59 62 
             self.add_commands("commands", cmds)

  • buildstream/scriptelement.py
    ... ... @@ -35,7 +35,8 @@ implementations.
    35 35 
     import os
    36 36 
     from collections import OrderedDict
    37 37
    38 
    -from . import Element, ElementError, Scope, SandboxFlags
    38 
    +from . import Element, SandboxFlags
    39 
    +from ._sysroot_dependency_loader import SysrootDependencyLoader, SysrootHelper
    39 40
    40 41
    41 42 
     class ScriptElement(Element):
    ... ... @@ -43,7 +44,6 @@ class ScriptElement(Element):
    43 44 
         __cwd = "/"
    44 45 
         __root_read_only = False
    45 46 
         __commands = None
    46 
    -    __layout = []
    47 47
    48 48 
         # The compose element's output is its dependencies, so
    49 49 
         # we must rebuild if the dependencies change even when
    ... ... @@ -59,6 +59,15 @@ class ScriptElement(Element):
    59 59 
         # added, to reduce the potential for confusion
    60 60 
         BST_FORBID_SOURCES = True
    61 61
    62 
    +    COMMON_CONFIG_KEYS = SysrootHelper.CONFIG_KEYS
    63 
    +
    64 
    +    DEPENDENCY_LOADER = SysrootDependencyLoader
    65 
    +
    66 
    +    def configure(self, node):
    67 
    +
    68 
    +        self.__stage_all = True  # pylint: disable=attribute-defined-outside-init
    69 
    +        self.__sysroots = SysrootHelper(self, node)  # pylint: disable=attribute-defined-outside-init
    70 
    +
    62 71 
         def set_work_dir(self, work_dir=None):
    63 72 
             """Sets the working dir
    64 73
    ... ... @@ -134,14 +143,8 @@ class ScriptElement(Element):
    134 143 
                In the case that no element is specified, a read-write directory will
    135 144 
                be made available at the specified location.
    136 145 
             """
    137 
    -        #
    138 
    -        # Even if this is an empty list by default, make sure that its
    139 
    -        # instance data instead of appending stuff directly onto class data.
    140 
    -        #
    141 
    -        if not self.__layout:
    142 
    -            self.__layout = []
    143 
    -        self.__layout.append({"element": element,
    144 
    -                              "destination": destination})
    146 
    +        self.__stage_all = False  # pylint: disable=attribute-defined-outside-init
    147 
    +        self.__sysroots.layout_add(element, destination)
    145 148
    146 149 
         def add_commands(self, group_name, command_list):
    147 150 
             """Adds a list of commands under the group-name.
    ... ... @@ -164,32 +167,15 @@ class ScriptElement(Element):
    164 167 
                 self.__commands = OrderedDict()
    165 168 
             self.__commands[group_name] = command_list
    166 169
    167 
    -    def __validate_layout(self):
    168 
    -        if self.__layout:
    169 
    -            # Cannot proceeed if layout is used, but none are for "/"
    170 
    -            root_defined = any([(entry['destination'] == '/') for entry in self.__layout])
    171 
    -            if not root_defined:
    172 
    -                raise ElementError("{}: Using layout, but none are staged as '/'"
    173 
    -                                   .format(self))
    174 
    -
    175 
    -            # Cannot proceed if layout specifies an element that isn't part
    176 
    -            # of the dependencies.
    177 
    -            for item in self.__layout:
    178 
    -                if item['element']:
    179 
    -                    if not self.search(Scope.BUILD, item['element']):
    180 
    -                        raise ElementError("{}: '{}' in layout not found in dependencies"
    181 
    -                                           .format(self, item['element']))
    182 
    -
    183 170 
         def preflight(self):
    184 
    -        # The layout, if set, must make sense.
    185 
    -        self.__validate_layout()
    171 
    +        self.__sysroots.validate()
    186 172
    187 173 
         def get_unique_key(self):
    188 174 
             return {
    189 175 
                 'commands': self.__commands,
    190 176 
                 'cwd': self.__cwd,
    191 177 
                 'install-root': self.__install_root,
    192 
    -            'layout': self.__layout,
    178 
    +            'layout': self.__sysroots.get_unique_key(),
    193 179 
                 'root-read-only': self.__root_read_only
    194 180 
             }
    195 181
    ... ... @@ -201,72 +187,11 @@ class ScriptElement(Element):
    201 187 
             # Setup environment
    202 188 
             sandbox.set_environment(self.get_environment())
    203 189
    204 
    -        # Tell the sandbox to mount the install root
    205 
    -        directories = {self.__install_root: False}
    206 
    -
    207 
    -        # Mark the artifact directories in the layout
    208 
    -        for item in self.__layout:
    209 
    -            destination = item['destination']
    210 
    -            was_artifact = directories.get(destination, False)
    211 
    -            directories[destination] = item['element'] or was_artifact
    212 
    -
    213 
    -        for directory, artifact in directories.items():
    214 
    -            # Root does not need to be marked as it is always mounted
    215 
    -            # with artifact (unless explicitly marked non-artifact)
    216 
    -            if directory != '/':
    217 
    -                sandbox.mark_directory(directory, artifact=artifact)
    190 
    +        self.__sysroots.configure_sandbox(sandbox, [self.__install_root])
    218 191
    219 192 
         def stage(self, sandbox):
    220 193
    221 
    -        # Stage the elements, and run integration commands where appropriate.
    222 
    -        if not self.__layout:
    223 
    -            # if no layout set, stage all dependencies into /
    224 
    -            for build_dep in self.dependencies(Scope.BUILD, recurse=False):
    225 
    -                with self.timed_activity("Staging {} at /"
    226 
    -                                         .format(build_dep.name), silent_nested=True):
    227 
    -                    build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/")
    228 
    -
    229 
    -            with sandbox.batch(SandboxFlags.NONE):
    230 
    -                for build_dep in self.dependencies(Scope.BUILD, recurse=False):
    231 
    -                    with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
    232 
    -                        for dep in build_dep.dependencies(Scope.RUN):
    233 
    -                            dep.integrate(sandbox)
    234 
    -        else:
    235 
    -            # If layout, follow its rules.
    236 
    -            for item in self.__layout:
    237 
    -
    238 
    -                # Skip layout members which dont stage an element
    239 
    -                if not item['element']:
    240 
    -                    continue
    241 
    -
    242 
    -                element = self.search(Scope.BUILD, item['element'])
    243 
    -                if item['destination'] == '/':
    244 
    -                    with self.timed_activity("Staging {} at /".format(element.name),
    245 
    -                                             silent_nested=True):
    246 
    -                        element.stage_dependency_artifacts(sandbox, Scope.RUN)
    247 
    -                else:
    248 
    -                    with self.timed_activity("Staging {} at {}"
    249 
    -                                             .format(element.name, item['destination']),
    250 
    -                                             silent_nested=True):
    251 
    -                        virtual_dstdir = sandbox.get_virtual_directory()
    252 
    -                        virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True)
    253 
    -                        element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
    254 
    -
    255 
    -            with sandbox.batch(SandboxFlags.NONE):
    256 
    -                for item in self.__layout:
    257 
    -
    258 
    -                    # Skip layout members which dont stage an element
    259 
    -                    if not item['element']:
    260 
    -                        continue
    261 
    -
    262 
    -                    element = self.search(Scope.BUILD, item['element'])
    263 
    -
    264 
    -                    # Integration commands can only be run for elements staged to /
    265 
    -                    if item['destination'] == '/':
    266 
    -                        with self.timed_activity("Integrating {}".format(element.name),
    267 
    -                                                 silent_nested=True):
    268 
    -                            for dep in element.dependencies(Scope.RUN):
    269 
    -                                dep.integrate(sandbox)
    194 
    +        self.__sysroots.stage(sandbox, self.__stage_all)
    270 195
    271 196 
             install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep)
    272 197 
             sandbox.get_virtual_directory().descend(install_root_path_components, create=True)

  • doc/source/format_declaring.rst
    ... ... @@ -159,6 +159,64 @@ See :ref:`format_dependencies` for more information on the dependency model.
    159 159
    160 160 
        The ``runtime-depends`` configuration is available since :ref:`format version 14 <project_format_version>`
    161 161
    162 
    +Sysroot'ed dependencies
    163 
    +~~~~~~~~~~~~~~~~~~~~~~~
    164 
    +
    165 
    +Build elements and script elements can support sysroot'ed
    166 
    +dependencies.  Sysroot'ed dependencies are intended for bootstraping
    167 
    +base systems or cross-compiling.
    168 
    +
    169 
    +Because sysroot'ed dependencies are plugin specific, they are defined
    170 
    +within the plugin configuration node.
    171 
    +
    172 
    +.. code:: yaml
    173 
    +
    174 
    +   config:
    175 
    +     # Specify some sysroot'ed dependencies
    176 
    +     sysroots:
    177 
    +     - path: /sysroot
    178 
    +       depends:
    179 
    +       - element1.bst
    180 
    +       - element2.bst
    181 
    +
    182 
    +During build, or initialization of build shell, sysroot'ed build
    183 
    +dependencies will be staged in the given sysroot path instead of '/'
    184 
    +together with the runtime dependencies of those sysroot'ed build
    185 
    +dependencies.
    186 
    +
    187 
    +It is possible to end up with indirect runtime dependencies in
    188 
    +different sysroots if they are staged from build dependencies with
    189 
    +different sysroots. They will be staged multiple times.
    190 
    +
    191 
    +Sysroot paths only apply to build dependencies. It is not possible to
    192 
    +define runtime dependencies either with ``type: runtime`` or
    193 
    +``runtime-depends``. It is possible to use ``all`` dependencies, but
    194 
    +the sysroot part is only for the build part not the runtime.
    195 
    +
    196 
    +For example:
    197 
    +
    198 
    +.. code:: yaml
    199 
    +
    200 
    +   config:
    201 
    +     sysroots:
    202 
    +     - path: /sysroot
    203 
    +       depends:
    204 
    +       - element.bst
    205 
    +
    206 
    +is equivalent to:
    207 
    +
    208 
    +.. code:: yaml
    209 
    +
    210 
    +   config:
    211 
    +     runtime-depends:
    212 
    +     - element.bst
    213 
    +     sysroots:
    214 
    +     - path: /sysroot
    215 
    +       build-depends:
    216 
    +       - element.bst
    217 
    +
    218 
    +:ref:`Integration commands <public_integration>` are never executed for
    219 
    +sysroot'ed dependencies.
    162 220
    163 221 
     .. _format_sources:
    164 222

  • tests/sysroot_depends/project/elements/a.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +  - kind: local
    4 
    +    path: files/a

  • tests/sysroot_depends/project/elements/b.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +  - kind: local
    4 
    +    path: files/b

  • tests/sysroot_depends/project/elements/base.bst
    1 
    +kind: stack
    2 
    +depends:
    3 
    +- base/base-alpine.bst

  • tests/sysroot_depends/project/elements/base/base-alpine.bst
    1 
    +kind: import
    2 
    +
    3 
    +description: |
    4 
    +  Alpine Linux base for tests
    5 
    +
    6 
    +  Generated using the `tests/integration-tests/base/generate-base.sh` script.
    7 
    +
    8 
    +sources:
    9 
    +  - kind: tar
    10 
    +    url: alpine:integration-tests-base.v1.x86_64.tar.xz
    11 
    +    base-dir: ''
    12 
    +    ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639

  • tests/sysroot_depends/project/elements/compose-integration.bst
    1 
    +kind: compose
    2 
    +
    3 
    +sysroots:
    4 
    +- path: /sysroot
    5 
    +  build-depends:
    6 
    +  - integration.bst

  • tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst
    1 
    +kind: manual
    2 
    +
    3 
    +build-depends:
    4 
    +- base.bst
    5 
    +
    6 
    +variables:
    7 
    +  install-root: "/"
    8 
    +
    9 
    +config:
    10 
    +  sysroots:
    11 
    +  - path: /other-sysroot
    12 
    +    build-depends:
    13 
    +    - layer2.bst

  • tests/sysroot_depends/project/elements/compose-layers.bst
    1 
    +kind: compose
    2 
    +
    3 
    +build-depends:
    4 
    +- layer2.bst

  • tests/sysroot_depends/project/elements/integration.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- base.bst
    5 
    +
    6 
    +config:
    7 
    +  install-commands:
    8 
    +    - echo 0 >"%{install-root}/integrated.txt"
    9 
    +
    10 
    +public:
    11 
    +  bst:
    12 
    +    integration-commands:
    13 
    +    - echo 1 >/integrated.txt

  • tests/sysroot_depends/project/elements/layer1-files.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: files/layer1

  • tests/sysroot_depends/project/elements/layer1.bst
    1 
    +kind: stack
    2 
    +
    3 
    +depends:
    4 
    +- layer1-files.bst

  • tests/sysroot_depends/project/elements/layer2-files.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: files/layer2

  • tests/sysroot_depends/project/elements/layer2.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- layer2-files.bst
    5 
    +
    6 
    +build-depends:
    7 
    +- base.bst
    8 
    +
    9 
    +config:
    10 
    +  sysroots:
    11 
    +  - path: /sysroot
    12 
    +    depends:
    13 
    +    - layer1.bst
    14 
    +
    15 
    +  install-commands:
    16 
    +  - mkdir -p "%{install-root}"
    17 
    +  - |
    18 
    +    for file in /*; do
    19 
    +      if test -f "${file}"; then
    20 
    +        cp "${file}" "%{install-root}"
    21 
    +      fi
    22 
    +    done

  • tests/sysroot_depends/project/elements/manual-integration-runtime.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- base.bst
    5 
    +
    6 
    +config:
    7 
    +  sysroots:
    8 
    +  - path: /sysroot
    9 
    +    depends:
    10 
    +    - integration.bst
    11 
    +
    12 
    +  install-commands:
    13 
    +    - mkdir -p "%{install-root}"
    14 
    +    - echo dummy >"%{install-root}/dummy.txt"

  • tests/sysroot_depends/project/elements/manual-integration.bst
    1 
    +kind: manual
    2 
    +
    3 
    +build-depends:
    4 
    +- base.bst
    5 
    +
    6 
    +config:
    7 
    +  sysroots:
    8 
    +  - path: /sysroot
    9 
    +    build-depends:
    10 
    +    - integration.bst
    11 
    +
    12 
    +  install-commands:
    13 
    +  - mkdir -p "%{install-root}/sysroot"
    14 
    +  - if test -f /sysroot/integrated.txt; then cp /sysroot/integrated.txt "%{install-root}/sysroot"; fi
    15 
    +  - if test -f /integrated.txt; then cp /integrated.txt "%{install-root}"; fi

  • tests/sysroot_depends/project/elements/sysroot-integration.bst
    1 
    +kind: manual
    2 
    +
    3 
    +variables:
    4 
    +  install-root: "/"
    5 
    +
    6 
    +config:
    7 
    +  sysroots:
    8 
    +  - path: /sysroot
    9 
    +    build-depends:
    10 
    +    - integration.bst

  • tests/sysroot_depends/project/elements/target-variable.bst
    1 
    +kind: manual
    2 
    +
    3 
    +build-depends:
    4 
    +- base.bst
    5 
    +
    6 
    +variables:
    7 
    +  mydir: test
    8 
    +  install-root: "/path"
    9 
    +
    10 
    +config:
    11 
    +  sysroots:
    12 
    +  - path: "/path/%{mydir}"
    13 
    +    build-depends:
    14 
    +    - b.bst

  • tests/sysroot_depends/project/elements/target.bst
    1 
    +kind: manual
    2 
    +
    3 
    +build-depends:
    4 
    +- base.bst
    5 
    +- a.bst
    6 
    +
    7 
    +variables:
    8 
    +  install-root: '/'
    9 
    +
    10 
    +config:
    11 
    +  sysroots:
    12 
    +  - path: /sysroot
    13 
    +    build-depends:
    14 
    +    - b.bst

  • tests/sysroot_depends/project/files/a/a.txt
    1 
    +test

  • tests/sysroot_depends/project/files/b/b.txt
    1 
    +test

  • tests/sysroot_depends/project/files/layer1/1
    1 
    +1

  • tests/sysroot_depends/project/files/layer2/2
    1 
    +2

  • tests/sysroot_depends/project/project.conf
    1 
    +name: test
    2 
    +element-path: elements
    3 
    +aliases:
    4 
    +  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
    5 
    +options:
    6 
    +  linux:
    7 
    +    type: bool
    8 
    +    description: Whether to expect a linux platform
    9 
    +    default: True

  • tests/sysroot_depends/sysroot_depends.py
    1 
    +import os
    2 
    +import pytest
    3 
    +from tests.testutils import cli_integration as cli
    4 
    +from tests.testutils.site import IS_LINUX, HAVE_BWRAP
    5 
    +
    6 
    +
    7 
    +# Project directory
    8 
    +DATA_DIR = os.path.join(
    9 
    +    os.path.dirname(os.path.realpath(__file__)),
    10 
    +    "project",
    11 
    +)
    12 
    +
    13 
    +
    14 
    +@pytest.mark.integration
    15 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    16 
    +@pytest.mark.datafiles(DATA_DIR)
    17 
    +def test_sysroot_dependency_smoke_test(datafiles, cli, tmpdir):
    18 
    +    "Test simple sysroot use case without integration"
    19 
    +
    20 
    +    project = str(datafiles)
    21 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    22 
    +
    23 
    +    result = cli.run(project=project,
    24 
    +                     args=['build', 'target.bst'])
    25 
    +    result.assert_success()
    26 
    +
    27 
    +    result = cli.run(project=project,
    28 
    +                     args=['checkout', 'target.bst', checkout])
    29 
    +    result.assert_success()
    30 
    +    assert os.path.exists(os.path.join(checkout, 'a.txt'))
    31 
    +    assert os.path.exists(os.path.join(checkout, 'sysroot', 'b.txt'))
    32 
    +
    33 
    +
    34 
    +@pytest.mark.integration
    35 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    36 
    +@pytest.mark.datafiles(DATA_DIR)
    37 
    +def test_skip_integration_commands_build_element(datafiles, cli, tmpdir):
    38 
    +    "Integration commands are not run on sysroots"
    39 
    +
    40 
    +    project = str(datafiles)
    41 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    42 
    +
    43 
    +    result = cli.run(project=project,
    44 
    +                     args=['build', 'manual-integration.bst'])
    45 
    +    result.assert_success()
    46 
    +
    47 
    +    result = cli.run(project=project,
    48 
    +                     args=['checkout', 'manual-integration.bst', checkout])
    49 
    +    result.assert_success()
    50 
    +
    51 
    +    sysroot_integrated = os.path.join(checkout, 'sysroot', 'integrated.txt')
    52 
    +    integrated = os.path.join(checkout, 'integrated.txt')
    53 
    +    assert os.path.exists(sysroot_integrated)
    54 
    +    with open(sysroot_integrated, 'r') as f:
    55 
    +        assert f.read() == '0\n'
    56 
    +    # We need to make sure that integration command has not been run on / either.
    57 
    +    assert not os.path.exists(integrated)
    58 
    +
    59 
    +
    60 
    +@pytest.mark.integration
    61 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    62 
    +@pytest.mark.datafiles(DATA_DIR)
    63 
    +def test_sysroot_only_for_build(cli, tmpdir, datafiles):
    64 
    +    project = str(datafiles)
    65 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    66 
    +
    67 
    +    result = cli.run(project=project,
    68 
    +                     args=['build', 'compose-layers.bst'])
    69 
    +    result.assert_success()
    70 
    +
    71 
    +    result = cli.run(project=project,
    72 
    +                     args=['checkout', 'compose-layers.bst', checkout])
    73 
    +
    74 
    +    result.assert_success()
    75 
    +    assert os.path.exists(os.path.join(checkout, '1'))
    76 
    +    assert os.path.exists(os.path.join(checkout, '2'))
    77 
    +    assert not os.path.exists(os.path.join(checkout, 'sysroot', '1'))
    78 
    +
    79 
    +
    80 
    +@pytest.mark.integration
    81 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    82 
    +@pytest.mark.datafiles(DATA_DIR)
    83 
    +def test_sysroot_only_for_build_with_sysroot(cli, tmpdir, datafiles):
    84 
    +    project = str(datafiles)
    85 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    86 
    +
    87 
    +    result = cli.run(project=project,
    88 
    +                     args=['build', 'compose-layers-with-sysroot.bst'])
    89 
    +    result.assert_success()
    90 
    +
    91 
    +    result = cli.run(project=project,
    92 
    +                     args=['checkout', 'compose-layers-with-sysroot.bst', checkout])
    93 
    +
    94 
    +    result.assert_success()
    95 
    +    assert os.path.exists(os.path.join(checkout, 'other-sysroot', '1'))
    96 
    +    assert os.path.exists(os.path.join(checkout, 'other-sysroot', '2'))
    97 
    +    assert not os.path.exists(os.path.join(checkout, 'sysroot', '1'))
    98 
    +
    99 
    +
    100 
    +@pytest.mark.integration
    101 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    102 
    +@pytest.mark.datafiles(DATA_DIR)
    103 
    +def test_shell_no_sysroot(cli, tmpdir, datafiles):
    104 
    +    "bst shell does not have sysroots and dependencies are integrated"
    105 
    +
    106 
    +    project = str(datafiles)
    107 
    +
    108 
    +    result = cli.run(project=project,
    109 
    +                     args=['build', 'base.bst', 'manual-integration-runtime.bst'])
    110 
    +    result.assert_success()
    111 
    +
    112 
    +    result = cli.run(project=project,
    113 
    +                     args=['shell', 'manual-integration-runtime.bst', '--', 'cat', '/integrated.txt'])
    114 
    +    result.assert_success()
    115 
    +    assert result.output == '1\n'
    116 
    +
    117 
    +    result = cli.run(project=project,
    118 
    +                     args=['shell', 'manual-integration-runtime.bst', '--', 'ls', '/sysroot/integrated.txt'])
    119 
    +    assert result.exit_code != 0
    120 
    +    assert result.output == ''
    121 
    +
    122 
    +
    123 
    +@pytest.mark.integration
    124 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    125 
    +@pytest.mark.datafiles(DATA_DIR)
    126 
    +def test_shell_build_sysroot(cli, tmpdir, datafiles):
    127 
    +    "Build shell should stage build dependencies sysroot'ed non integrated"
    128 
    +
    129 
    +    project = str(datafiles)
    130 
    +
    131 
    +    result = cli.run(project=project,
    132 
    +                     args=['build', 'base.bst', 'integration.bst'])
    133 
    +    result.assert_success()
    134 
    +
    135 
    +    result = cli.run(project=project,
    136 
    +                     args=['shell', '-b', 'manual-integration.bst', '--', 'cat', '/sysroot/integrated.txt'])
    137 
    +    result.assert_success()
    138 
    +    assert result.output == '0\n'
    139 
    +
    140 
    +
    141 
    +@pytest.mark.integration
    142 
    +@pytest.mark.datafiles(DATA_DIR)
    143 
    +def test_show_dependencies_only_once(cli, tmpdir, datafiles):
    144 
    +    """Dependencies should not show up in status several times when they
    145 
    +    are staged with multiple sysroots"""
    146 
    +
    147 
    +    project = str(datafiles)
    148 
    +
    149 
    +    result = cli.run(project=project,
    150 
    +                     args=['show', '--format', '%{name}', 'manual-integration.bst'])
    151 
    +    result.assert_success()
    152 
    +    pipeline = result.output.splitlines()
    153 
    +    assert pipeline == ['base/base-alpine.bst',
    154 
    +                        'base.bst',
    155 
    +                        'integration.bst',
    156 
    +                        'manual-integration.bst']
    157 
    +
    158 
    +
    159 
    +@pytest.mark.integration
    160 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    161 
    +@pytest.mark.datafiles(DATA_DIR)
    162 
    +def test_sysroot_path_subst_variable(datafiles, cli, tmpdir):
    163 
    +    "Test that variables are expanded in sysroot path"
    164 
    +
    165 
    +    project = str(datafiles)
    166 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    167 
    +
    168 
    +    result = cli.run(project=project,
    169 
    +                     args=['build', 'target-variable.bst'])
    170 
    +    result.assert_success()
    171 
    +
    172 
    +    result = cli.run(project=project,
    173 
    +                     args=['checkout', 'target-variable.bst', checkout])
    174 
    +    result.assert_success()
    175 
    +
    176 
    +    assert os.path.exists(os.path.join(checkout, 'test', 'b.txt'))

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]