GNOME.org
 

[Notes] [Git][BuildStream/buildstream][phil/cache-key-stability-test] 8 commits: tox.ini: Specify minimum version of click-man

Title: GitLab

Phil Dawson pushed to branch phil/cache-key-stability-test at BuildStream / buildstream

Commits:

10 changed files:

Changes:

  • buildstream/_loader/loadelement.py
    ... ... @@ -39,6 +39,20 @@ from .types import Symbol, Dependency
    39 39 
     #    loader (Loader): The Loader object for this element
    40 40 
     #
    41 41 
     class LoadElement():
    42 
    +    # Dependency():
    43 
    +    #
    44 
    +    # A link from a LoadElement to its dependencies.
    45 
    +    #
    46 
    +    # Keeps a link to one of the current Element's dependencies, together with
    47 
    +    # its dependency type.
    48 
    +    #
    49 
    +    # Args:
    50 
    +    #    element (LoadElement): a LoadElement on which there is a dependency
    51 
    +    #    dep_type (str): the type of dependency this dependency link is
    52 
    +    class Dependency:
    53 
    +        def __init__(self, element, dep_type):
    54 
    +            self.element = element
    55 
    +            self.dep_type = dep_type
    42 56
    43 57 
         def __init__(self, node, filename, loader):
    44 58
    ... ... @@ -74,8 +88,11 @@ class LoadElement():
    74 88 
                 'build-depends', 'runtime-depends',
    75 89 
             ])
    76 90
    77 
    -        # Extract the Dependencies
    78 
    -        self.deps = _extract_depends_from_node(self.node)
    91 
    +        self.dependencies = []
    92 
    +
    93 
    +    @property
    94 
    +    def junction(self):
    95 
    +        return self._loader.project.junction
    79 96
    80 97 
         # depends():
    81 98 
         #
    ... ... @@ -101,8 +118,8 @@ class LoadElement():
    101 118 
                 return
    102 119
    103 120 
             self._dep_cache = {}
    104 
    -        for dep in self.deps:
    105 
    -            elt = self._loader.get_element_for_dep(dep)
    121 
    +        for dep in self.dependencies:
    122 
    +            elt = dep.element
    106 123
    107 124 
                 # Ensure the cache of the element we depend on
    108 125 
                 elt._ensure_depends_cache()

  • buildstream/_loader/loader.py
    ... ... @@ -19,7 +19,6 @@
    19 19
    20 20 
     import os
    21 21 
     from functools import cmp_to_key
    22 
    -from collections import namedtuple
    23 22 
     from collections.abc import Mapping
    24 23 
     import tempfile
    25 24 
     import shutil
    ... ... @@ -32,8 +31,8 @@ from .._profile import Topics, profile_start, profile_end
    32 31 
     from .._includes import Includes
    33 32 
     from .._yamlcache import YamlCache
    34 33
    35 
    -from .types import Symbol, Dependency
    36 
    -from .loadelement import LoadElement
    34 
    +from .types import Symbol
    35 
    +from .loadelement import LoadElement, _extract_depends_from_node
    37 36 
     from . import MetaElement
    38 37 
     from . import MetaSource
    39 38 
     from ..types import CoreWarnings
    ... ... @@ -112,7 +111,7 @@ class Loader():
    112 111
    113 112 
             # First pass, recursively load files and populate our table of LoadElements
    114 113 
             #
    115 
    -        deps = []
    114 
    +        target_elements = []
    116 115
    117 116 
             # XXX This will need to be changed to the context's top-level project if this method
    118 117 
             # is ever used for subprojects
    ... ... @@ -122,10 +121,10 @@ class Loader():
    122 121 
             with YamlCache.open(self._context, cache_file) as yaml_cache:
    123 122 
                 for target in targets:
    124 123 
                     profile_start(Topics.LOAD_PROJECT, target)
    125 
    -                junction, name, loader = self._parse_name(target, rewritable, ticker,
    126 
    -                                                          fetch_subprojects=fetch_subprojects)
    127 
    -                loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    128 
    -                deps.append(Dependency(name, junction=junction))
    124 
    +                _junction, name, loader = self._parse_name(target, rewritable, ticker,
    125 
    +                                                           fetch_subprojects=fetch_subprojects)
    126 
    +                element = loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    127 
    +                target_elements.append(element)
    129 128 
                     profile_end(Topics.LOAD_PROJECT, target)
    130 129
    131 130 
             #
    ... ... @@ -134,29 +133,29 @@ class Loader():
    134 133
    135 134 
             # Set up a dummy element that depends on all top-level targets
    136 135 
             # to resolve potential circular dependencies between them
    137 
    -        DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])
    138 
    -
    139 
    -        dummy = DummyTarget(name='', full_name='', deps=deps)
    140 
    -        self._elements[''] = dummy
    136 
    +        dummy_target = LoadElement("", "", self)
    137 
    +        dummy_target.dependencies.extend(
    138 
    +            LoadElement.Dependency(element, Symbol.RUNTIME)
    139 
    +            for element in target_elements
    140 
    +        )
    141 141
    142 142 
             profile_key = "_".join(t for t in targets)
    143 143 
             profile_start(Topics.CIRCULAR_CHECK, profile_key)
    144 
    -        self._check_circular_deps('')
    144 
    +        self._check_circular_deps(dummy_target)
    145 145 
             profile_end(Topics.CIRCULAR_CHECK, profile_key)
    146 146
    147 147 
             ret = []
    148 148 
             #
    149 149 
             # Sort direct dependencies of elements by their dependency ordering
    150 150 
             #
    151 
    -        for target in targets:
    152 
    -            profile_start(Topics.SORT_DEPENDENCIES, target)
    153 
    -            junction, name, loader = self._parse_name(target, rewritable, ticker,
    154 
    -                                                      fetch_subprojects=fetch_subprojects)
    155 
    -            loader._sort_dependencies(name)
    156 
    -            profile_end(Topics.SORT_DEPENDENCIES, target)
    151 
    +        for element in target_elements:
    152 
    +            loader = element._loader
    153 
    +            profile_start(Topics.SORT_DEPENDENCIES, element.name)
    154 
    +            loader._sort_dependencies(element)
    155 
    +            profile_end(Topics.SORT_DEPENDENCIES, element.name)
    157 156 
                 # Finally, wrap what we have into LoadElements and return the target
    158 157 
                 #
    159 
    -            ret.append(loader._collect_element(name))
    158 
    +            ret.append(loader._collect_element(element))
    160 159
    161 160 
             return ret
    162 161
    ... ... @@ -184,22 +183,6 @@ class Loader():
    184 183 
                 if os.path.exists(self._tempdir):
    185 184 
                     shutil.rmtree(self._tempdir)
    186 185
    187 
    -    # get_element_for_dep():
    188 
    -    #
    189 
    -    # Gets a cached LoadElement by Dependency object
    190 
    -    #
    191 
    -    # This is used by LoadElement
    192 
    -    #
    193 
    -    # Args:
    194 
    -    #    dep (Dependency): The dependency to search for
    195 
    -    #
    196 
    -    # Returns:
    197 
    -    #    (LoadElement): The cached LoadElement
    198 
    -    #
    199 
    -    def get_element_for_dep(self, dep):
    200 
    -        loader = self._get_loader_for_dep(dep)
    201 
    -        return loader._elements[dep.name]
    202 
    -
    203 186 
         ###########################################
    204 187 
         #            Private Methods              #
    205 188 
         ###########################################
    ... ... @@ -272,8 +255,10 @@ class Loader():
    272 255
    273 256 
             self._elements[filename] = element
    274 257
    258 
    +        dependencies = _extract_depends_from_node(node)
    259 
    +
    275 260 
             # Load all dependency files for the new LoadElement
    276 
    -        for dep in element.deps:
    261 
    +        for dep in dependencies:
    277 262 
                 if dep.junction:
    278 263 
                     self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
    279 264 
                     loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
    ... ... @@ -288,7 +273,9 @@ class Loader():
    288 273 
                                     "{}: Cannot depend on junction"
    289 274 
                                     .format(dep.provenance))
    290 275
    291 
    -        deps_names = [dep.name for dep in element.deps]
    276 
    +            element.dependencies.append(LoadElement.Dependency(dep_element, dep.dep_type))
    277 
    +
    278 
    +        deps_names = [dep.name for dep in dependencies]
    292 279 
             self._warn_invalid_elements(deps_names)
    293 280
    294 281 
             return element
    ... ... @@ -299,12 +286,12 @@ class Loader():
    299 286 
         # dependencies already resolved.
    300 287 
         #
    301 288 
         # Args:
    302 
    -    #    element_name (str): The element-path relative element name to check
    289 
    +    #    element (str): The element to check
    303 290 
         #
    304 291 
         # Raises:
    305 292 
         #    (LoadError): In case there was a circular dependency error
    306 293 
         #
    307 
    -    def _check_circular_deps(self, element_name, check_elements=None, validated=None, sequence=None):
    294 
    +    def _check_circular_deps(self, element, check_elements=None, validated=None, sequence=None):
    308 295
    309 296 
             if check_elements is None:
    310 297 
                 check_elements = {}
    ... ... @@ -313,38 +300,31 @@ class Loader():
    313 300 
             if sequence is None:
    314 301 
                 sequence = []
    315 302
    316 
    -        element = self._elements[element_name]
    317 
    -
    318 
    -        # element name must be unique across projects
    319 
    -        # to be usable as key for the check_elements and validated dicts
    320 
    -        element_name = element.full_name
    321 
    -
    322 303 
             # Skip already validated branches
    323 
    -        if validated.get(element_name) is not None:
    304 
    +        if validated.get(element) is not None:
    324 305 
                 return
    325 306
    326 
    -        if check_elements.get(element_name) is not None:
    307 
    +        if check_elements.get(element) is not None:
    327 308 
                 # Create `chain`, the loop of element dependencies from this
    328 309 
                 # element back to itself, by trimming everything before this
    329 310 
                 # element from the sequence under consideration.
    330 
    -            chain = sequence[sequence.index(element_name):]
    331 
    -            chain.append(element_name)
    311 
    +            chain = sequence[sequence.index(element.full_name):]
    312 
    +            chain.append(element.full_name)
    332 313 
                 raise LoadError(LoadErrorReason.CIRCULAR_DEPENDENCY,
    333 314 
                                 ("Circular dependency detected at element: {}\n" +
    334 315 
                                  "Dependency chain: {}")
    335 
    -                            .format(element.name, " -> ".join(chain)))
    316 
    +                            .format(element.full_name, " -> ".join(chain)))
    336 317
    337 318 
             # Push / Check each dependency / Pop
    338 
    -        check_elements[element_name] = True
    339 
    -        sequence.append(element_name)
    340 
    -        for dep in element.deps:
    341 
    -            loader = self._get_loader_for_dep(dep)
    342 
    -            loader._check_circular_deps(dep.name, check_elements, validated, sequence)
    343 
    -        del check_elements[element_name]
    319 
    +        check_elements[element] = True
    320 
    +        sequence.append(element.full_name)
    321 
    +        for dep in element.dependencies:
    322 
    +            dep.element._loader._check_circular_deps(dep.element, check_elements, validated, sequence)
    323 
    +        del check_elements[element]
    344 324 
             sequence.pop()
    345 325
    346 326 
             # Eliminate duplicate paths
    347 
    -        validated[element_name] = True
    327 
    +        validated[element] = True
    348 328
    349 329 
         # _sort_dependencies():
    350 330 
         #
    ... ... @@ -357,28 +337,21 @@ class Loader():
    357 337 
         # sorts throughout the build process.
    358 338 
         #
    359 339 
         # Args:
    360 
    -    #    element_name (str): The element-path relative element name to sort
    340 
    +    #    element (LoadElement): The element to sort
    361 341 
         #
    362 
    -    def _sort_dependencies(self, element_name, visited=None):
    342 
    +    def _sort_dependencies(self, element, visited=None):
    363 343 
             if visited is None:
    364 
    -            visited = {}
    344 
    +            visited = set()
    365 345
    366 
    -        element = self._elements[element_name]
    367 
    -
    368 
    -        # element name must be unique across projects
    369 
    -        # to be usable as key for the visited dict
    370 
    -        element_name = element.full_name
    371 
    -
    372 
    -        if visited.get(element_name) is not None:
    346 
    +        if element in visited:
    373 347 
                 return
    374 348
    375 
    -        for dep in element.deps:
    376 
    -            loader = self._get_loader_for_dep(dep)
    377 
    -            loader._sort_dependencies(dep.name, visited=visited)
    349 
    +        for dep in element.dependencies:
    350 
    +            dep.element._loader._sort_dependencies(dep.element, visited=visited)
    378 351
    379 352 
             def dependency_cmp(dep_a, dep_b):
    380 
    -            element_a = self.get_element_for_dep(dep_a)
    381 
    -            element_b = self.get_element_for_dep(dep_b)
    353 
    +            element_a = dep_a.element
    354 
    +            element_b = dep_b.element
    382 355
    383 356 
                 # Sort on inter element dependency first
    384 357 
                 if element_a.depends(element_b):
    ... ... @@ -395,21 +368,21 @@ class Loader():
    395 368 
                         return -1
    396 369
    397 370 
                 # All things being equal, string comparison.
    398 
    -            if dep_a.name > dep_b.name:
    371 
    +            if element_a.name > element_b.name:
    399 372 
                     return 1
    400 
    -            elif dep_a.name < dep_b.name:
    373 
    +            elif element_a.name < element_b.name:
    401 374 
                     return -1
    402 375
    403 376 
                 # Sort local elements before junction elements
    404 377 
                 # and use string comparison between junction elements
    405 
    -            if dep_a.junction and dep_b.junction:
    406 
    -                if dep_a.junction > dep_b.junction:
    378 
    +            if element_a.junction and element_b.junction:
    379 
    +                if element_a.junction > element_b.junction:
    407 380 
                         return 1
    408 
    -                elif dep_a.junction < dep_b.junction:
    381 
    +                elif element_a.junction < element_b.junction:
    409 382 
                         return -1
    410 
    -            elif dep_a.junction:
    383 
    +            elif element_a.junction:
    411 384 
                     return -1
    412 
    -            elif dep_b.junction:
    385 
    +            elif element_b.junction:
    413 386 
                     return 1
    414 387
    415 388 
                 # This wont ever happen
    ... ... @@ -418,26 +391,23 @@ class Loader():
    418 391 
             # Now dependency sort, we ensure that if any direct dependency
    419 392 
             # directly or indirectly depends on another direct dependency,
    420 393 
             # it is found later in the list.
    421 
    -        element.deps.sort(key=cmp_to_key(dependency_cmp))
    394 
    +        element.dependencies.sort(key=cmp_to_key(dependency_cmp))
    422 395
    423 
    -        visited[element_name] = True
    396 
    +        visited.add(element)
    424 397
    425 398 
         # _collect_element()
    426 399 
         #
    427 400 
         # Collect the toplevel elements we have
    428 401 
         #
    429 402 
         # Args:
    430 
    -    #    element_name (str): The element-path relative element name to sort
    403 
    +    #    element (LoadElement): The element for which to load a MetaElement
    431 404 
         #
    432 405 
         # Returns:
    433 406 
         #    (MetaElement): A recursively loaded MetaElement
    434 407 
         #
    435 
    -    def _collect_element(self, element_name):
    436 
    -
    437 
    -        element = self._elements[element_name]
    438 
    -
    408 
    +    def _collect_element(self, element):
    439 409 
             # Return the already built one, if we already built it
    440 
    -        meta_element = self._meta_elements.get(element_name)
    410 
    +        meta_element = self._meta_elements.get(element.name)
    441 411 
             if meta_element:
    442 412 
                 return meta_element
    443 413
    ... ... @@ -461,10 +431,10 @@ class Loader():
    461 431 
                     del source[Symbol.DIRECTORY]
    462 432
    463 433 
                 index = sources.index(source)
    464 
    -            meta_source = MetaSource(element_name, index, element_kind, kind, source, directory)
    434 
    +            meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
    465 435 
                 meta_sources.append(meta_source)
    466 436
    467 
    -        meta_element = MetaElement(self.project, element_name, element_kind,
    437 
    +        meta_element = MetaElement(self.project, element.name, element_kind,
    468 438 
                                        elt_provenance, meta_sources,
    469 439 
                                        _yaml.node_get(node, Mapping, Symbol.CONFIG, default_value={}),
    470 440 
                                        _yaml.node_get(node, Mapping, Symbol.VARIABLES, default_value={}),
    ... ... @@ -475,12 +445,12 @@ class Loader():
    475 445 
                                        element_kind == 'junction')
    476 446
    477 447 
             # Cache it now, make sure it's already there before recursing
    478 
    -        self._meta_elements[element_name] = meta_element
    448 
    +        self._meta_elements[element.name] = meta_element
    479 449
    480 450 
             # Descend
    481 
    -        for dep in element.deps:
    482 
    -            loader = self._get_loader_for_dep(dep)
    483 
    -            meta_dep = loader._collect_element(dep.name)
    451 
    +        for dep in element.dependencies:
    452 
    +            loader = dep.element._loader
    453 
    +            meta_dep = loader._collect_element(dep.element)
    484 454 
                 if dep.dep_type != 'runtime':
    485 455 
                     meta_element.build_dependencies.append(meta_dep)
    486 456 
                 if dep.dep_type != 'build':
    ... ... @@ -539,7 +509,7 @@ class Loader():
    539 509 
                     return None
    540 510
    541 511 
             # meta junction element
    542 
    -        meta_element = self._collect_element(filename)
    512 
    +        meta_element = self._collect_element(self._elements[filename])
    543 513 
             if meta_element.kind != 'junction':
    544 514 
                 raise LoadError(LoadErrorReason.INVALID_DATA,
    545 515 
                                 "{}: Expected junction but element kind is {}".format(filename, meta_element.kind))
    ... ... @@ -601,23 +571,6 @@ class Loader():
    601 571
    602 572 
             return loader
    603 573
    604 
    -    # _get_loader_for_dep():
    605 
    -    #
    606 
    -    # Gets the appropriate Loader for a Dependency object
    607 
    -    #
    608 
    -    # Args:
    609 
    -    #    dep (Dependency): A Dependency object
    610 
    -    #
    611 
    -    # Returns:
    612 
    -    #    (Loader): The Loader object to use for this Dependency
    613 
    -    #
    614 
    -    def _get_loader_for_dep(self, dep):
    615 
    -        if dep.junction:
    616 
    -            # junction dependency, delegate to appropriate loader
    617 
    -            return self._loaders[dep.junction]
    618 
    -        else:
    619 
    -            return self
    620 
    -
    621 574 
         # _parse_name():
    622 575 
         #
    623 576 
         # Get junction and base name of element along with loader for the sub-project

  • tests/cachekey/cachekey.py
    ... ... @@ -214,3 +214,41 @@ def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings,
    214 214 
         second_keys = run_get_cache_key("second", second_warnings)
    215 215
    216 216 
         assert compare_cache_keys(first_keys, second_keys) == identical_keys
    217 
    +
    218 
    +
    219 
    +@pytest.mark.datafiles(DATA_DIR)
    220 
    +def test_keys_stable_over_targets(cli, datafiles):
    221 
    +    root_element = 'elements/key-stability/top-level.bst'
    222 
    +    target1 = 'elements/key-stability/t1.bst'
    223 
    +    target2 = 'elements/key-stability/t2.bst'
    224 
    +
    225 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    226 
    +    full_graph_result = cli.run(project=project, args=[
    227 
    +        'show',
    228 
    +        '--format', '%{name}::%{full-key}',
    229 
    +        root_element
    230 
    +    ])
    231 
    +    full_graph_result.assert_success()
    232 
    +    all_cache_keys = parse_output_keys(full_graph_result.output)
    233 
    +
    234 
    +    ordering1_result = cli.run(project=project, args=[
    235 
    +        'show',
    236 
    +        '--format', '%{name}::%{full-key}',
    237 
    +        target1,
    238 
    +        target2
    239 
    +    ])
    240 
    +    ordering1_result.assert_success()
    241 
    +    ordering1_cache_keys = parse_output_keys(ordering1_result.output)
    242 
    +
    243 
    +    ordering2_result = cli.run(project=project, args=[
    244 
    +        'show',
    245 
    +        '--format', '%{name}::%{full-key}',
    246 
    +        target2,
    247 
    +        target1
    248 
    +    ])
    249 
    +    ordering2_result.assert_success()
    250 
    +    ordering2_cache_keys = parse_output_keys(ordering2_result.output)
    251 
    +
    252 
    +    for element in ordering1_cache_keys:
    253 
    +        assert ordering1_cache_keys[element] == ordering2_cache_keys[element]
    254 
    +        assert ordering1_cache_keys[element] == all_cache_keys[element]

  • tests/cachekey/project/elements/key-stability/aaa.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: elements/key-stability/aaa.bst

  • tests/cachekey/project/elements/key-stability/t1.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: elements/key-stability/t1.bst
    5 
    +depends:
    6 
    +- elements/key-stability/zzz.bst

  • tests/cachekey/project/elements/key-stability/t2.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: elements/key-stability/t2.bst
    5 
    +depends:
    6 
    +- elements/key-stability/aaa.bst
    7 
    +- elements/key-stability/zzz.bst

  • tests/cachekey/project/elements/key-stability/top-level.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: elements/key-stability/top-level.bst
    5 
    +depends:
    6 
    +- elements/key-stability/t1.bst
    7 
    +- elements/key-stability/t2.bst

  • tests/cachekey/project/elements/key-stability/zzz.bst
    1 
    +kind: import
    2 
    +sources:
    3 
    +- kind: local
    4 
    +  path: elements/key-stability/zzz.bst

  • tests/testutils/site.py
    ... ... @@ -18,7 +18,7 @@ try:
    18 18 
         utils.get_host_tool('git')
    19 19 
         HAVE_GIT = True
    20 20 
         out = str(subprocess.check_output(['git', '--version']), "utf-8")
    21 
    -    version = tuple(int(x) for x in out.split(' ', 2)[2].split('.'))
    21 
    +    version = tuple(int(x) for x in out.split(' ')[2].split('.'))
    22 22 
         HAVE_OLD_GIT = version < (1, 8, 5)
    23 23 
     except ProgramNotFoundError:
    24 24 
         HAVE_GIT = False

  • tox.ini
    ... ... @@ -88,5 +88,5 @@ whitelist_externals =
    88 88 
     commands =
    89 89 
         python3 setup.py --command-packages=click_man.commands man_pages
    90 90 
     deps =
    91 
    -    click-man
    91 
    +    click-man >= 0.3.0
    92 92 
         -rrequirements/requirements.txt

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]