GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/sysroot_dependencies] Add support for importing dependencies in sysroots.

Title: GitLab

Valentin David pushed to branch valentindavid/sysroot_dependencies at BuildStream / buildstream

Commits:

29 changed files:

Changes:

  • buildstream/_loader/loadelement.py
    ... ... @@ -72,10 +72,24 @@ class LoadElement():
    72 72 
                 'variables', 'environment', 'environment-nocache',
    73 73 
                 'config', 'public', 'description',
    74 74 
                 'build-depends', 'runtime-depends',
    75 
    +            'sysroots',
    75 76 
             ])
    76 77
    78 
    +        self.deps = []
    79 
    +        sysroots = _yaml.node_get(node, list, 'sysroots', default_value=[])
    80 
    +        for sysroot in sysroots:
    81 
    +            _yaml.node_validate(sysroot, ['path', 'depends', 'build-depends'])
    82 
    +            path = _yaml.node_get(sysroot, str, 'path')
    83 
    +            for dep in _extract_depends_from_node(sysroot):
    84 
    +                if dep.dep_type == Symbol.RUNTIME:
    85 
    +                    raise LoadError(LoadErrorReason.INVALID_DATA,
    86 
    +                                    "{}: Sysroot'ed dependencies can not be of type 'runtime'"
    87 
    +                                    .format(dep.provenance, dep_type))
    88 
    +                self.deps.append((path, dep))
    89 
    +
    77 90 
             # Extract the Dependencies
    78 
    -        self.deps = _extract_depends_from_node(self.node)
    91 
    +        for dep in _extract_depends_from_node(self.node):
    92 
    +            self.deps.append(('/', dep))
    79 93
    80 94 
         # depends():
    81 95 
         #
    ... ... @@ -101,7 +115,7 @@ class LoadElement():
    101 115 
                 return
    102 116
    103 117 
             self._dep_cache = {}
    104 
    -        for dep in self.deps:
    118 
    +        for _, dep in self.deps:
    105 119 
                 elt = self._loader.get_element_for_dep(dep)
    106 120
    107 121 
                 # Ensure the cache of the element we depend on

  • buildstream/_loader/loader.py
    ... ... @@ -121,7 +121,7 @@ class Loader():
    121 121 
                     junction, name, loader = self._parse_name(target, rewritable, ticker,
    122 122 
                                                               fetch_subprojects=fetch_subprojects)
    123 123 
                     loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    124 
    -                deps.append(Dependency(name, junction=junction))
    124 
    +                deps.append(('/', Dependency(name, junction=junction)))
    125 125 
                     profile_end(Topics.LOAD_PROJECT, target)
    126 126
    127 127 
             #
    ... ... @@ -269,7 +269,7 @@ class Loader():
    269 269 
             self._elements[filename] = element
    270 270
    271 271 
             # Load all dependency files for the new LoadElement
    272 
    -        for dep in element.deps:
    272 
    +        for _, dep in element.deps:
    273 273 
                 if dep.junction:
    274 274 
                     self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
    275 275 
                     loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
    ... ... @@ -286,6 +286,7 @@ class Loader():
    286 286
    287 287 
             return element
    288 288
    289 
    +
    289 290 
         # _check_circular_deps():
    290 291 
         #
    291 292 
         # Detect circular dependencies on LoadElements with
    ... ... @@ -330,7 +331,7 @@ class Loader():
    330 331 
             # Push / Check each dependency / Pop
    331 332 
             check_elements[element_name] = True
    332 333 
             sequence.append(element_name)
    333 
    -        for dep in element.deps:
    334 
    +        for _, dep in element.deps:
    334 335 
                 loader = self._get_loader_for_dep(dep)
    335 336 
                 loader._check_circular_deps(dep.name, check_elements, validated, sequence)
    336 337 
             del check_elements[element_name]
    ... ... @@ -365,14 +366,21 @@ class Loader():
    365 366 
             if visited.get(element_name) is not None:
    366 367 
                 return
    367 368
    368 
    -        for dep in element.deps:
    369 
    +        for _, dep in element.deps:
    369 370 
                 loader = self._get_loader_for_dep(dep)
    370 371 
                 loader._sort_dependencies(dep.name, visited=visited)
    371 372
    372 
    -        def dependency_cmp(dep_a, dep_b):
    373 
    +        def dependency_cmp(sdep_a, sdep_b):
    374 
    +            sysroot_a, dep_a = sdep_a
    375 
    +            sysroot_b, dep_b = sdep_b
    373 376 
                 element_a = self.get_element_for_dep(dep_a)
    374 377 
                 element_b = self.get_element_for_dep(dep_b)
    375 378
    379 
    +            if sysroot_a < sysroot_b:
    380 
    +                return -1
    381 
    +            if sysroot_b < sysroot_a:
    382 
    +                return 1
    383 
    +
    376 384 
                 # Sort on inter element dependency first
    377 385 
                 if element_a.depends(element_b):
    378 386 
                     return 1
    ... ... @@ -471,11 +479,11 @@ class Loader():
    471 479 
             self._meta_elements[element_name] = meta_element
    472 480
    473 481 
             # Descend
    474 
    -        for dep in element.deps:
    482 
    +        for sysroot, dep in element.deps:
    475 483 
                 loader = self._get_loader_for_dep(dep)
    476 484 
                 meta_dep = loader._collect_element(dep.name)
    477 485 
                 if dep.dep_type != 'runtime':
    478 
    -                meta_element.build_dependencies.append(meta_dep)
    486 
    +                meta_element.build_dependencies.append((sysroot, meta_dep))
    479 487 
                 if dep.dep_type != 'build':
    480 488 
                     meta_element.dependencies.append(meta_dep)
    481 489

  • buildstream/buildelement.py
    ... ... @@ -213,9 +213,10 @@ class BuildElement(Element):
    213 213
    214 214 
             # Run any integration commands provided by the dependencies
    215 215 
             # once they are all staged and ready
    216 
    -        with self.timed_activity("Integrating sandbox"):
    217 
    -            for dep in self.dependencies(Scope.BUILD):
    218 
    -                dep.integrate(sandbox)
    216 
    +        with self.timed_activity("Integrating sandbox (buildelement)"):
    217 
    +            for sysroot, dep in self.dependencies(Scope.BUILD, with_sysroot=True):
    218 
    +                if sysroot == '/':
    219 
    +                    dep.integrate(sandbox)
    219 220
    220 221 
             # Stage sources in the build root
    221 222 
             self.stage_sources(sandbox, self.get_variable('build-root'))

  • buildstream/element.py
    ... ... @@ -375,7 +375,8 @@ class Element(Plugin):
    375 375 
             for source in self.__sources:
    376 376 
                 yield source
    377 377
    378 
    -    def dependencies(self, scope, *, recurse=True, visited=None, recursed=False):
    378 
    +    def dependencies(self, scope, *, recurse=True, visited=None, recursed=False,
    379 
    +                     with_sysroot=False, sysroot='/'):
    379 380 
             """dependencies(scope, *, recurse=True)
    380 381
    381 382 
             A generator function which yields the dependencies of the given element.
    ... ... @@ -400,40 +401,56 @@ class Element(Plugin):
    400 401
    401 402 
             scope_set = set((Scope.BUILD, Scope.RUN)) if scope == Scope.ALL else set((scope,))
    402 403
    403 
    -        if full_name in visited and scope_set.issubset(visited[full_name]):
    404 
    +        if with_sysroot:
    405 
    +            key = (sysroot, full_name)
    406 
    +        else:
    407 
    +            key = full_name
    408 
    +
    409 
    +        if key in visited and scope_set.issubset(visited[key]):
    404 410 
                 return
    405 411
    406 412 
             should_yield = False
    407 
    -        if full_name not in visited:
    408 
    -            visited[full_name] = scope_set
    413 
    +        if key not in visited:
    414 
    +            visited[key] = scope_set
    409 415 
                 should_yield = True
    410 416 
             else:
    411 
    -            visited[full_name] |= scope_set
    417 
    +            visited[key] |= scope_set
    412 418
    413 419 
             if recurse or not recursed:
    414 420 
                 if scope == Scope.ALL:
    415 
    -                for dep in self.__build_dependencies:
    421 
    +                build_deps = []
    422 
    +                for dep_sysroot, dep in self.__build_dependencies:
    423 
    +                    new_sysroot = self._subst_string(dep_sysroot) if not recursed else sysroot
    416 424 
                         yield from dep.dependencies(Scope.ALL, recurse=recurse,
    417 
    -                                                visited=visited, recursed=True)
    425 
    +                                                visited=visited, recursed=True,
    426 
    +                                                sysroot=new_sysroot, with_sysroot=with_sysroot)
    427 
    +                    build_deps.append(dep)
    418 428
    419 429 
                     for dep in self.__runtime_dependencies:
    420 
    -                    if dep not in self.__build_dependencies:
    430 
    +                    if dep not in build_deps:
    421 431 
                             yield from dep.dependencies(Scope.ALL, recurse=recurse,
    422 
    -                                                    visited=visited, recursed=True)
    432 
    +                                                    visited=visited, recursed=True,
    433 
    +                                                    sysroot=sysroot, with_sysroot=with_sysroot)
    423 434
    424 435 
                 elif scope == Scope.BUILD:
    425 
    -                for dep in self.__build_dependencies:
    436 
    +                for dep_sysroot, dep in self.__build_dependencies:
    437 
    +                    new_sysroot = self._subst_string(dep_sysroot) if not recursed else sysroot
    426 438 
                         yield from dep.dependencies(Scope.RUN, recurse=recurse,
    427 
    -                                                visited=visited, recursed=True)
    439 
    +                                                visited=visited, recursed=True,
    440 
    +                                                sysroot=new_sysroot, with_sysroot=with_sysroot)
    428 441
    429 442 
                 elif scope == Scope.RUN:
    430 443 
                     for dep in self.__runtime_dependencies:
    431 444 
                         yield from dep.dependencies(Scope.RUN, recurse=recurse,
    432 
    -                                                visited=visited, recursed=True)
    445 
    +                                                visited=visited, recursed=True,
    446 
    +                                                sysroot=sysroot, with_sysroot=with_sysroot)
    433 447
    434 448 
             # Yeild self only at the end, after anything needed has been traversed
    435 
    -        if should_yield and (recurse or recursed) and (scope == Scope.ALL or scope == Scope.RUN):
    436 
    -            yield self
    449 
    +        if should_yield and (recurse or recursed) and (scope in [Scope.ALL, Scope.RUN]):
    450 
    +            if with_sysroot:
    451 
    +                yield sysroot, self
    452 
    +            else:
    453 
    +                yield self
    437 454
    438 455 
         def search(self, scope, name):
    439 456 
             """Search for a dependency by name
    ... ... @@ -632,7 +649,7 @@ class Element(Plugin):
    632 649 
                 vbasedir = sandbox.get_virtual_directory()
    633 650 
                 vstagedir = vbasedir \
    634 651 
                     if path is None \
    635 
    -                else vbasedir.descend(path.lstrip(os.sep).split(os.sep))
    652 
    +                else vbasedir.descend(path.lstrip(os.sep).split(os.sep), create=True)
    636 653
    637 654 
                 files = list(self.__compute_splits(include, exclude, orphans))
    638 655
    ... ... @@ -650,7 +667,8 @@ class Element(Plugin):
    650 667 
             return link_result.combine(copy_result)
    651 668
    652 669 
         def stage_dependency_artifacts(self, sandbox, scope, *, path=None,
    653 
    -                                   include=None, exclude=None, orphans=True):
    670 
    +                                   include=None, exclude=None, orphans=True,
    671 
    +                                   build=True):
    654 672 
             """Stage element dependencies in scope
    655 673
    656 674 
             This is primarily a convenience wrapper around
    ... ... @@ -680,7 +698,14 @@ class Element(Plugin):
    680 698 
             if self.__can_build_incrementally() and workspace.last_successful:
    681 699 
                 old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful)
    682 700
    683 
    -        for dep in self.dependencies(scope):
    701 
    +        def deps():
    702 
    +            if build:
    703 
    +                yield from self.dependencies(scope, with_sysroot=True)
    704 
    +            else:
    705 
    +                for dep in self.dependencies(scope, with_sysroot=False):
    706 
    +                    yield '/', dep
    707 
    +
    708 
    +        for sysroot, dep in deps():
    684 709 
                 # If we are workspaced, and we therefore perform an
    685 710 
                 # incremental build, we must ensure that we update the mtimes
    686 711 
                 # of any files created by our dependencies since the last
    ... ... @@ -705,8 +730,13 @@ class Element(Plugin):
    705 730 
                         if utils._is_main_process():
    706 731 
                             self._get_context().get_workspaces().save_config()
    707 732
    733 
    +            if build:
    734 
    +                sub_path = os.path.join(path, os.path.relpath(sysroot, '/')) if path else sysroot
    735 
    +            else:
    736 
    +                sub_path = path
    737 
    +
    708 738 
                 result = dep.stage_artifact(sandbox,
    709 
    -                                        path=path,
    739 
    +                                        path=sub_path,
    710 740 
                                             include=include,
    711 741 
                                             exclude=exclude,
    712 742 
                                             orphans=orphans,
    ... ... @@ -906,9 +936,9 @@ class Element(Plugin):
    906 936 
             for meta_dep in meta.dependencies:
    907 937 
                 dependency = Element._new_from_meta(meta_dep, artifacts)
    908 938 
                 element.__runtime_dependencies.append(dependency)
    909 
    -        for meta_dep in meta.build_dependencies:
    939 
    +        for sysroot, meta_dep in meta.build_dependencies:
    910 940 
                 dependency = Element._new_from_meta(meta_dep, artifacts)
    911 
    -            element.__build_dependencies.append(dependency)
    941 
    +            element.__build_dependencies.append((sysroot, dependency))
    912 942
    913 943 
             return element
    914 944
    ... ... @@ -1088,14 +1118,11 @@ class Element(Plugin):
    1088 1118 
                 # Weak cache key includes names of direct build dependencies
    1089 1119 
                 # but does not include keys of dependencies.
    1090 1120 
                 if self.BST_STRICT_REBUILD:
    1091 
    -                dependencies = [
    1092 
    -                    e._get_cache_key(strength=_KeyStrength.WEAK)
    1093 
    -                    for e in self.dependencies(Scope.BUILD)
    1094 
    -                ]
    1121 
    +                dependencies = [(sysroot, e._get_cache_key(strength=_KeyStrength.WEAK))
    1122 
    +                                for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
    1095 1123 
                 else:
    1096 
    -                dependencies = [
    1097 
    -                    e.name for e in self.dependencies(Scope.BUILD, recurse=False)
    1098 
    -                ]
    1124 
    +                dependencies = [(sysroot, e.name)
    1125 
    +                                for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
    1099 1126
    1100 1127 
                 self.__weak_cache_key = self.__calculate_cache_key(dependencies)
    1101 1128
    ... ... @@ -1123,9 +1150,8 @@ class Element(Plugin):
    1123 1150 
                         return
    1124 1151
    1125 1152 
             if self.__strict_cache_key is None:
    1126 
    -            dependencies = [
    1127 
    -                e.__strict_cache_key for e in self.dependencies(Scope.BUILD)
    1128 
    -            ]
    1153 
    +            dependencies = [(sysroot, e.__strict_cache_key)
    1154 
    +                            for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
    1129 1155 
                 self.__strict_cache_key = self.__calculate_cache_key(dependencies)
    1130 1156
    1131 1157 
                 if self.__strict_cache_key is None:
    ... ... @@ -1165,10 +1191,8 @@ class Element(Plugin):
    1165 1191 
                     strong_key, _ = self.__get_artifact_metadata_keys()
    1166 1192 
                     self.__cache_key = strong_key
    1167 1193 
                 elif self.__assemble_scheduled or self.__assemble_done:
    1168 
    -                # Artifact will or has been built, not downloaded
    1169 
    -                dependencies = [
    1170 
    -                    e._get_cache_key() for e in self.dependencies(Scope.BUILD)
    1171 
    -                ]
    1194 
    +                dependencies = [(sysroot, e._get_cache_key())
    1195 
    +                                for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
    1172 1196 
                     self.__cache_key = self.__calculate_cache_key(dependencies)
    1173 1197
    1174 1198 
                 if self.__cache_key is None:
    ... ... @@ -1331,7 +1355,7 @@ class Element(Plugin):
    1331 1355 
                         # Stage deps in the sandbox root
    1332 1356 
                         if deps == 'run':
    1333 1357 
                             with self.timed_activity("Staging dependencies", silent_nested=True):
    1334 
    -                            self.stage_dependency_artifacts(sandbox, scope)
    1358 
    +                            self.stage_dependency_artifacts(sandbox, scope, build=False)
    1335 1359
    1336 1360 
                             # Run any integration commands provided by the dependencies
    1337 1361 
                             # once they are all staged and ready
    ... ... @@ -2024,8 +2048,11 @@ class Element(Plugin):
    2024 2048 
         #
    2025 2049 
         def __calculate_cache_key(self, dependencies):
    2026 2050 
             # No cache keys for dependencies which have no cache keys
    2027 
    -        if None in dependencies:
    2028 
    -            return None
    2051 
    +        for dep in dependencies:
    2052 
    +            if dep[1] is None:
    2053 
    +                return None
    2054 
    +        # Do not break cache keys
    2055 
    +        dependencies = [(sysroot, key) if sysroot != '/' else key for sysroot, key in dependencies]
    2029 2056
    2030 2057 
             # Generate dict that is used as base for all cache keys
    2031 2058 
             if self.__cache_key_dict is None:

  • buildstream/plugins/elements/compose.py
    ... ... @@ -115,15 +115,16 @@ class ComposeElement(Element):
    115 115 
             # Run any integration commands provided by the dependencies
    116 116 
             # once they are all staged and ready
    117 117 
             if self.integration:
    118 
    -            with self.timed_activity("Integrating sandbox"):
    118 
    +            with self.timed_activity("Integrating sandbox (compose)"):
    119 119 
                     if require_split:
    120 120
    121 121 
                         # Make a snapshot of all the files before integration-commands are run.
    122 122 
                         snapshot = set(vbasedir.list_relative_paths())
    123 123 
                         vbasedir.mark_unmodified()
    124 124
    125 
    -                for dep in self.dependencies(Scope.BUILD):
    126 
    -                    dep.integrate(sandbox)
    125 
    +                for sysroot, dep in self.dependencies(Scope.BUILD, with_sysroot=True):
    126 
    +                    if sysroot == '/':
    127 
    +                        dep.integrate(sandbox)
    127 128
    128 129 
                     if require_split:
    129 130 
                         # Calculate added, modified and removed files

  • doc/source/format_declaring.rst
    ... ... @@ -159,6 +159,57 @@ See :ref:`format_dependencies` for more information on the dependency model.
    159 159
    160 160 
        The ``runtime-depends`` configuration is available since :ref:`format version 14 <project_format_version>`
    161 161
    162 
    +Sysroot'ed dependencies
    163 
    +~~~~~~~~~~~~~~~~~~~~~~~
    164 
    +
    165 
    +Sysroot'ed dependencies are intended for bootstraping base systems or
    166 
    +cross-compiling.
    167 
    +
    168 
    +.. code:: yaml
    169 
    +
    170 
    +   # Specify some sysroot'ed dependencies
    171 
    +   sysroots:
    172 
    +   - path: /sysroot
    173 
    +     depends:
    174 
    +     - element1.bst
    175 
    +     - element2.bst
    176 
    +
    177 
    +During build, or initialization of build shell, sysroot'ed build
    178 
    +dependencies will be staged in the given sysroot path instead of '/'
    179 
    +together with the runtime dependencies of those sysroot'ed build
    180 
    +dependencies.
    181 
    +
    182 
    +It is possible to end up with indirect runtime dependencies in
    183 
    +different sysroots if they are staged from build dependencies with
    184 
    +different sysroots. They will be staged multiple times.
    185 
    +
    186 
    +Sysroot path only apply to build dependencies. It is not possible to
    187 
    +define runtime dependencies neither with ``type: runtime`` nor
    188 
    +``runtime-depends``. It is possible to use ``all`` dependencies, but
    189 
    +the sysroot part is only for the build part not the runtime.
    190 
    +
    191 
    +For example:
    192 
    +
    193 
    +.. code:: yaml
    194 
    +
    195 
    +   sysroots:
    196 
    +   - path: /sysroot
    197 
    +     depends:
    198 
    +     - element.bst
    199 
    +
    200 
    +is equivalent to:
    201 
    +
    202 
    +.. code:: yaml
    203 
    +
    204 
    +   runtime-depends:
    205 
    +   - element.bst
    206 
    +   sysroots:
    207 
    +   - path: /sysroot
    208 
    +     build-depends:
    209 
    +     - element.bst
    210 
    +
    211 
    +:ref:`Integration commands <public_integration>` are never executed for
    212 
    +sysroot'ed dependencies.
    162 213
    163 214 
     .. _format_sources:
    164 215

  • tests/loader/dependencies.py
    ... ... @@ -27,8 +27,8 @@ def test_two_files(datafiles):
    27 27
    28 28 
         assert(len(element.dependencies) == 1)
    29 29 
         firstdep = element.dependencies[0]
    30 
    -    assert(isinstance(firstdep, MetaElement))
    31 
    -    assert(firstdep.kind == 'manual')
    30 
    +    assert(isinstance(firstdep[1], MetaElement))
    31 
    +    assert(firstdep[1].kind == 'manual')
    32 32
    33 33
    34 34 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -47,22 +47,22 @@ def test_shared_dependency(datafiles):
    47 47 
         # The first specified dependency is 'thefirstdep'
    48 48 
         #
    49 49 
         firstdep = element.dependencies[0]
    50 
    -    assert(isinstance(firstdep, MetaElement))
    51 
    -    assert(firstdep.kind == 'manual')
    52 
    -    assert(len(firstdep.dependencies) == 0)
    50 
    +    assert(isinstance(firstdep[1], MetaElement))
    51 
    +    assert(firstdep[1].kind == 'manual')
    52 
    +    assert(len(firstdep[1].dependencies) == 0)
    53 53
    54 54 
         # The second specified dependency is 'shareddep'
    55 55 
         #
    56 56 
         shareddep = element.dependencies[1]
    57 
    -    assert(isinstance(shareddep, MetaElement))
    58 
    -    assert(shareddep.kind == 'shareddep')
    59 
    -    assert(len(shareddep.dependencies) == 1)
    57 
    +    assert(isinstance(shareddep[1], MetaElement))
    58 
    +    assert(shareddep[1].kind == 'shareddep')
    59 
    +    assert(len(shareddep[1].dependencies) == 1)
    60 60
    61 61 
         # The element which shareddep depends on is
    62 62 
         # the same element in memory as firstdep
    63 63 
         #
    64 
    -    shareddepdep = shareddep.dependencies[0]
    65 
    -    assert(isinstance(shareddepdep, MetaElement))
    64 
    +    shareddepdep = shareddep[1].dependencies[0]
    65 
    +    assert(isinstance(shareddepdep[1], MetaElement))
    66 66
    67 67 
         # Assert they are in fact the same LoadElement
    68 68 
         #
    ... ... @@ -71,7 +71,7 @@ def test_shared_dependency(datafiles):
    71 71 
         # equality test with '==' which is one of those operator
    72 72 
         # overridable thingies.
    73 73 
         #
    74 
    -    assert(shareddepdep is firstdep)
    74 
    +    assert(shareddepdep[1] is firstdep[1])
    75 75
    76 76
    77 77 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -86,8 +86,8 @@ def test_dependency_dict(datafiles):
    86 86
    87 87 
         assert(len(element.dependencies) == 1)
    88 88 
         firstdep = element.dependencies[0]
    89 
    -    assert(isinstance(firstdep, MetaElement))
    90 
    -    assert(firstdep.kind == 'manual')
    89 
    +    assert(isinstance(firstdep[1], MetaElement))
    90 
    +    assert(firstdep[1].kind == 'manual')
    91 91
    92 92
    93 93 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -135,7 +135,7 @@ def test_build_dependency(datafiles):
    135 135
    136 136 
         assert(len(element.build_dependencies) == 1)
    137 137 
         firstdep = element.build_dependencies[0]
    138 
    -    assert(isinstance(firstdep, MetaElement))
    138 
    +    assert(isinstance(firstdep[1], MetaElement))
    139 139
    140 140 
         assert(len(element.dependencies) == 0)
    141 141
    ... ... @@ -151,7 +151,7 @@ def test_runtime_dependency(datafiles):
    151 151
    152 152 
         assert(len(element.dependencies) == 1)
    153 153 
         firstdep = element.dependencies[0]
    154 
    -    assert(isinstance(firstdep, MetaElement))
    154 
    +    assert(isinstance(firstdep[1], MetaElement))
    155 155
    156 156 
         assert(len(element.build_dependencies) == 0)
    157 157
    ... ... @@ -168,9 +168,9 @@ def test_build_runtime_dependency(datafiles):
    168 168 
         assert(len(element.dependencies) == 1)
    169 169 
         assert(len(element.build_dependencies) == 1)
    170 170 
         firstdep = element.dependencies[0]
    171 
    -    assert(isinstance(firstdep, MetaElement))
    171 
    +    assert(isinstance(firstdep[1], MetaElement))
    172 172 
         firstbuilddep = element.build_dependencies[0]
    173 
    -    assert(firstdep == firstbuilddep)
    173 
    +    assert(firstdep[1] == firstbuilddep[1])
    174 174
    175 175
    176 176 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -185,9 +185,9 @@ def test_all_dependency(datafiles):
    185 185 
         assert(len(element.dependencies) == 1)
    186 186 
         assert(len(element.build_dependencies) == 1)
    187 187 
         firstdep = element.dependencies[0]
    188 
    -    assert(isinstance(firstdep, MetaElement))
    188 
    +    assert(isinstance(firstdep[1], MetaElement))
    189 189 
         firstbuilddep = element.build_dependencies[0]
    190 
    -    assert(firstdep == firstbuilddep)
    190 
    +    assert(firstdep[1] == firstbuilddep[1])
    191 191
    192 192
    193 193 
     @pytest.mark.datafiles(DATA_DIR)

  • tests/sysroot_depends/project/elements/a.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +  - kind: local
    4 
    +    path: files/a

  • tests/sysroot_depends/project/elements/b.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +  - kind: local
    4 
    +    path: files/b

  • tests/sysroot_depends/project/elements/base.bst
    1 
    +kind: stack
    2 
    +depends:
    3 
    +- base/base-alpine.bst

  • tests/sysroot_depends/project/elements/base/base-alpine.bst
    1 
    +kind: import
    2 
    +
    3 
    +description: |
    4 
    +  Alpine Linux base for tests
    5 
    +
    6 
    +  Generated using the `tests/integration-tests/base/generate-base.sh` script.
    7 
    +
    8 
    +sources:
    9 
    +  - kind: tar
    10 
    +    url: alpine:integration-tests-base.v1.x86_64.tar.xz
    11 
    +    base-dir: ''
    12 
    +    ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639

  • tests/sysroot_depends/project/elements/compose-integration.bst
    1 
    +kind: compose
    2 
    +
    3 
    +sysroots:
    4 
    +- path: /sysroot
    5 
    +  build-depends:
    6 
    +  - integration.bst

  • tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst
    1 
    +kind: compose
    2 
    +
    3 
    +sysroots:
    4 
    +- path: /other-sysroot
    5 
    +  build-depends:
    6 
    +  - layer2.bst

  • tests/sysroot_depends/project/elements/compose-layers.bst
    1 
    +kind: compose
    2 
    +
    3 
    +build-depends:
    4 
    +- layer2.bst

  • tests/sysroot_depends/project/elements/integration.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- base.bst
    5 
    +
    6 
    +config:
    7 
    +  install-commands:
    8 
    +    - echo 0 >"%{install-root}/integrated.txt"
    9 
    +
    10 
    +public:
    11 
    +  bst:
    12 
    +    integration-commands:
    13 
    +    - echo 1 >/integrated.txt

  • tests/sysroot_depends/project/elements/layer1-files.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: files/layer1

  • tests/sysroot_depends/project/elements/layer1.bst
    1 
    +kind: stack
    2 
    +
    3 
    +depends:
    4 
    +- layer1-files.bst

  • tests/sysroot_depends/project/elements/layer2-files.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: files/layer2

  • tests/sysroot_depends/project/elements/layer2.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- layer2-files.bst
    5 
    +
    6 
    +build-depends:
    7 
    +- base.bst
    8 
    +
    9 
    +sysroots:
    10 
    +- path: /sysroot
    11 
    +  depends:
    12 
    +  - layer1.bst
    13 
    +
    14 
    +config:
    15 
    +  install-commands:
    16 
    +  - mkdir -p "%{install-root}"
    17 
    +  - |
    18 
    +    for file in /*; do
    19 
    +      if test -f "${file}"; then
    20 
    +        cp "${file}" "%{install-root}"
    21 
    +      fi
    22 
    +    done

  • tests/sysroot_depends/project/elements/manual-integration-runtime.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +- base.bst
    5 
    +
    6 
    +sysroots:
    7 
    +- path: /sysroot
    8 
    +  depends:
    9 
    +  - integration.bst
    10 
    +
    11 
    +config:
    12 
    +  install-commands:
    13 
    +    - mkdir -p "%{install-root}"
    14 
    +    - echo dummy >"%{install-root}/dummy.txt"

  • tests/sysroot_depends/project/elements/manual-integration.bst
    1 
    +kind: manual
    2 
    +
    3 
    +build-depends:
    4 
    +- base.bst
    5 
    +
    6 
    +sysroots:
    7 
    +- path: /sysroot
    8 
    +  build-depends:
    9 
    +  - integration.bst
    10 
    +
    11 
    +config:
    12 
    +  install-commands:
    13 
    +    - mkdir -p "%{install-root}/sysroot"
    14 
    +    - if test -f /sysroot/integrated.txt; then cp /sysroot/integrated.txt "%{install-root}/sysroot"; fi
    15 
    +    - if test -f /integrated.txt; then cp /integrated.txt "%{install-root}"; fi

  • tests/sysroot_depends/project/elements/target-variable.bst
    1 
    +kind: compose
    2 
    +
    3 
    +build-depends:
    4 
    +- a.bst
    5 
    +
    6 
    +variables:
    7 
    +  mydir: test
    8 
    +
    9 
    +sysroots:
    10 
    +- path: "/path/%{mydir}"
    11 
    +  build-depends:
    12 
    +  - b.bst

  • tests/sysroot_depends/project/elements/target.bst
    1 
    +kind: compose
    2 
    +
    3 
    +build-depends:
    4 
    +- a.bst
    5 
    +
    6 
    +sysroots:
    7 
    +- path: /sysroot
    8 
    +  build-depends:
    9 
    +  - b.bst

  • tests/sysroot_depends/project/files/a/a.txt
    1 
    +test

  • tests/sysroot_depends/project/files/b/b.txt
    1 
    +test

  • tests/sysroot_depends/project/files/layer1/1
    1 
    +1

  • tests/sysroot_depends/project/files/layer2/2
    1 
    +2

  • tests/sysroot_depends/project/project.conf
    1 
    +name: test
    2 
    +element-path: elements
    3 
    +aliases:
    4 
    +  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/

  • tests/sysroot_depends/sysroot_depends.py
    1 
    +import os
    2 
    +import pytest
    3 
    +from tests.testutils import cli_integration as cli
    4 
    +
    5 
    +
    6 
    +# Project directory
    7 
    +DATA_DIR = os.path.join(
    8 
    +    os.path.dirname(os.path.realpath(__file__)),
    9 
    +    "project",
    10 
    +)
    11 
    +
    12 
    +
    13 
    +@pytest.mark.datafiles(DATA_DIR)
    14 
    +def test_sysroot_dependency_smoke_test(datafiles, cli, tmpdir):
    15 
    +    "Test simple sysroot use case without integration"
    16 
    +
    17 
    +    project = str(datafiles)
    18 
    +    checkout = os.path.join(tmpdir, 'checkout')
    19 
    +
    20 
    +    result = cli.run(project=project,
    21 
    +                     args=['build', 'target.bst'])
    22 
    +    result.assert_success()
    23 
    +
    24 
    +    result = cli.run(project=project,
    25 
    +                     args=['checkout', 'target.bst', checkout])
    26 
    +    result.assert_success()
    27 
    +    assert os.path.exists(os.path.join(checkout, 'a.txt'))
    28 
    +    assert os.path.exists(os.path.join(checkout, 'sysroot', 'b.txt'))
    29 
    +
    30 
    +
    31 
    +@pytest.mark.integration
    32 
    +@pytest.mark.datafiles(DATA_DIR)
    33 
    +def test_skip_integration_commands_compose(datafiles, cli, tmpdir):
    34 
    +    "Integration commands are not run on sysroots"
    35 
    +
    36 
    +    project = str(datafiles)
    37 
    +    checkout = os.path.join(tmpdir, 'checkout')
    38 
    +
    39 
    +    result = cli.run(project=project,
    40 
    +                     args=['build', 'compose-integration.bst'])
    41 
    +    result.assert_success()
    42 
    +
    43 
    +    result = cli.run(project=project,
    44 
    +                     args=['checkout', 'compose-integration.bst', checkout])
    45 
    +    result.assert_success()
    46 
    +
    47 
    +    integrated = os.path.join(checkout, 'sysroot', 'integrated.txt')
    48 
    +    assert os.path.exists(integrated)
    49 
    +    with open(integrated, 'r') as f:
    50 
    +        assert f.read() == '0\n'
    51 
    +
    52 
    +
    53 
    +@pytest.mark.integration
    54 
    +@pytest.mark.datafiles(DATA_DIR)
    55 
    +def test_skip_integration_commands_build_element(datafiles, cli, tmpdir):
    56 
    +    "Integration commands are not run on sysroots"
    57 
    +
    58 
    +    project = str(datafiles)
    59 
    +    checkout = os.path.join(tmpdir, 'checkout')
    60 
    +
    61 
    +    result = cli.run(project=project,
    62 
    +                     args=['build', 'manual-integration.bst'])
    63 
    +    result.assert_success()
    64 
    +
    65 
    +    result = cli.run(project=project,
    66 
    +                     args=['checkout', 'manual-integration.bst', checkout])
    67 
    +    result.assert_success()
    68 
    +
    69 
    +    sysroot_integrated = os.path.join(checkout, 'sysroot', 'integrated.txt')
    70 
    +    integrated = os.path.join(checkout, 'integrated.txt')
    71 
    +    assert os.path.exists(sysroot_integrated)
    72 
    +    with open(sysroot_integrated, 'r') as f:
    73 
    +        assert f.read() == '0\n'
    74 
    +    # We need to make sure that integration command has not been run on / either.
    75 
    +    assert not os.path.exists(integrated)
    76 
    +
    77 
    +
    78 
    +@pytest.mark.integration
    79 
    +@pytest.mark.datafiles(DATA_DIR)
    80 
    +def test_sysroot_only_for_build(cli, tmpdir, datafiles):
    81 
    +    project = str(datafiles)
    82 
    +    checkout = os.path.join(tmpdir, 'checkout')
    83 
    +
    84 
    +    result = cli.run(project=project,
    85 
    +                     args=['build', 'compose-layers.bst'])
    86 
    +    result.assert_success()
    87 
    +
    88 
    +    result = cli.run(project=project,
    89 
    +                     args=['checkout', 'compose-layers.bst', checkout])
    90 
    +
    91 
    +    result.assert_success()
    92 
    +    assert os.path.exists(os.path.join(checkout, '1'))
    93 
    +    assert os.path.exists(os.path.join(checkout, '2'))
    94 
    +    assert not os.path.exists(os.path.join(checkout, 'sysroot', '1'))
    95 
    +
    96 
    +
    97 
    +@pytest.mark.integration
    98 
    +@pytest.mark.datafiles(DATA_DIR)
    99 
    +def test_sysroot_only_for_build_with_sysroot(cli, tmpdir, datafiles):
    100 
    +    project = str(datafiles)
    101 
    +    checkout = os.path.join(tmpdir, 'checkout')
    102 
    +
    103 
    +    result = cli.run(project=project,
    104 
    +                     args=['build', 'compose-layers-with-sysroot.bst'])
    105 
    +    result.assert_success()
    106 
    +
    107 
    +    result = cli.run(project=project,
    108 
    +                     args=['checkout', 'compose-layers-with-sysroot.bst', checkout])
    109 
    +
    110 
    +    result.assert_success()
    111 
    +    assert os.path.exists(os.path.join(checkout, 'other-sysroot', '1'))
    112 
    +    assert os.path.exists(os.path.join(checkout, 'other-sysroot', '2'))
    113 
    +    assert not os.path.exists(os.path.join(checkout, 'sysroot', '1'))
    114 
    +
    115 
    +
    116 
    +@pytest.mark.integration
    117 
    +@pytest.mark.datafiles(DATA_DIR)
    118 
    +def test_shell_no_sysroot(cli, tmpdir, datafiles):
    119 
    +    "bst shell does not have sysroots and dependencies are integrated"
    120 
    +
    121 
    +    project = str(datafiles)
    122 
    +
    123 
    +    result = cli.run(project=project,
    124 
    +                     args=['build', 'base.bst', 'manual-integration-runtime.bst'])
    125 
    +    result.assert_success()
    126 
    +
    127 
    +    result = cli.run(project=project,
    128 
    +                     args=['shell', 'manual-integration-runtime.bst', '--', 'cat', '/integrated.txt'])
    129 
    +    result.assert_success()
    130 
    +    assert result.output == '1\n'
    131 
    +
    132 
    +    result = cli.run(project=project,
    133 
    +                     args=['shell', 'manual-integration-runtime.bst', '--', 'ls', '/sysroot/integrated.txt'])
    134 
    +    assert result.exit_code != 0
    135 
    +    assert result.output == ''
    136 
    +
    137 
    +
    138 
    +@pytest.mark.integration
    139 
    +@pytest.mark.datafiles(DATA_DIR)
    140 
    +def test_shell_build_sysroot(cli, tmpdir, datafiles):
    141 
    +    "Build shell should stage build dependencies sysroot'ed non integrated"
    142 
    +
    143 
    +    project = str(datafiles)
    144 
    +
    145 
    +    result = cli.run(project=project,
    146 
    +                     args=['build', 'base.bst', 'integration.bst'])
    147 
    +    result.assert_success()
    148 
    +
    149 
    +    result = cli.run(project=project,
    150 
    +                     args=['shell', '-b', 'manual-integration.bst', '--', 'cat', '/sysroot/integrated.txt'])
    151 
    +    result.assert_success()
    152 
    +    assert result.output == '0\n'
    153 
    +
    154 
    +
    155 
    +@pytest.mark.integration
    156 
    +@pytest.mark.datafiles(DATA_DIR)
    157 
    +def test_show_dependencies_only_once(cli, tmpdir, datafiles):
    158 
    +    """Dependencies should not show up in status several times when they
    159 
    +    are staged with multiple sysroots"""
    160 
    +
    161 
    +    project = str(datafiles)
    162 
    +
    163 
    +    result = cli.run(project=project,
    164 
    +                     args=['show', '--format', '%{name}', 'manual-integration.bst'])
    165 
    +    result.assert_success()
    166 
    +    pipeline = result.output.splitlines()
    167 
    +    assert pipeline == ['base/base-alpine.bst',
    168 
    +                        'base.bst',
    169 
    +                        'integration.bst',
    170 
    +                        'manual-integration.bst']
    171 
    +
    172 
    +
    173 
    +@pytest.mark.datafiles(DATA_DIR)
    174 
    +def test_sysroot_path_subst_variable(datafiles, cli, tmpdir):
    175 
    +    "Test that variables are expanded in sysroot path"
    176 
    +
    177 
    +    project = str(datafiles)
    178 
    +    checkout = os.path.join(tmpdir, 'checkout')
    179 
    +
    180 
    +    result = cli.run(project=project,
    181 
    +                     args=['build', 'target.bst'])
    182 
    +    result.assert_success()
    183 
    +
    184 
    +    result = cli.run(project=project,
    185 
    +                     args=['checkout', 'target.bst', checkout])
    186 
    +    result.assert_success()
    187 
    +    assert os.path.exists(os.path.join(checkout, 'a.txt'))
    188 
    +    assert os.path.exists(os.path.join(checkout, '/path/test', 'b.txt'))

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]