GNOME.org
 

[Notes] [Git][BuildStream/buildstream][bschubert/rework-sort] fixup! Add reverse dependencies in the LoadElement

Title: GitLab

Benjamin Schubert pushed to branch bschubert/rework-sort at BuildStream / buildstream

Commits:

4 changed files:

Changes:

  • buildstream/_loader/loadelement.py
    ... ... @@ -63,7 +63,7 @@ class LoadElement():
    63 63 
             self.node = node       # The YAML node
    64 64 
             self.name = filename   # The element name
    65 65 
             self.full_name = None  # The element full name (with associated junction)
    66 
    -        self.deps = None       # The list of Dependency objects
    66 
    +        self.visited = False
    67 67
    68 68 
             #
    69 69 
             # Private members

  • buildstream/_loader/loader.py
    ... ... @@ -19,6 +19,7 @@
    19 19
    20 20 
     import os
    21 21 
     from functools import cmp_to_key
    22 
    +from collections import defaultdict, deque
    22 23 
     from collections.abc import Mapping
    23 24 
     import tempfile
    24 25 
     import shutil
    ... ... @@ -144,18 +145,23 @@ class Loader():
    144 145 
             self._check_circular_deps(dummy_target)
    145 146 
             profile_end(Topics.CIRCULAR_CHECK, profile_key)
    146 147
    147 
    -        ret = []
    148 148 
             #
    149 
    -        # Sort direct dependencies of elements by their dependency ordering
    149 
    +        # Load and sort direct dependencies of elements by their dependency ordering
    150 150 
             #
    151 
    -        for element in target_elements:
    152 
    -            loader = element._loader
    153 
    -            profile_start(Topics.SORT_DEPENDENCIES, element.name)
    154 
    -            loader._sort_dependencies(element)
    155 
    -            profile_end(Topics.SORT_DEPENDENCIES, element.name)
    156 
    -            # Finally, wrap what we have into LoadElements and return the target
    157 
    -            #
    158 
    -            ret.append(loader._collect_element(element))
    151 
    +        key = "-".join([e.name for e in target_elements])
    152 
    +        profile_start(Topics.COLLECT_ELEMENTS, key)
    153 
    +        ret = loader._collect_elements(target_elements)
    154 
    +        profile_end(Topics.COLLECT_ELEMENTS, key)
    155 
    +
    156 
    +        print("#" * 60)
    157 
    +        for e in ret:
    158 
    +            print(e.name)
    159 
    +            print("dependencies:")
    160 
    +            for d in e.dependencies:
    161 
    +                print("\t", d.name)
    162 
    +            print("build deps:")
    163 
    +            for d in e.build_dependencies:
    164 
    +                print("\t", d.name)
    159 165
    160 166 
             return ret
    161 167
    ... ... @@ -448,17 +454,194 @@ class Loader():
    448 454 
             # Cache it now, make sure it's already there before recursing
    449 455 
             self._meta_elements[element.name] = meta_element
    450 456
    451 
    -        # Descend
    452 
    -        for dep in element.dependencies:
    453 
    -            loader = dep.element._loader
    454 
    -            meta_dep = loader._collect_element(dep.element)
    455 
    -            if dep.dep_type != 'runtime':
    456 
    -                meta_element.build_dependencies.append(meta_dep)
    457 
    -            if dep.dep_type != 'build':
    458 
    -                meta_element.dependencies.append(meta_dep)
    457 
    +        # # Descend
    458 
    +        # for dep in element.dependencies:
    459 
    +        #     loader = dep.element._loader
    460 
    +        #     meta_dep = loader._collect_element(dep.element)
    461 
    +        #     if dep.dep_type != 'runtime':
    462 
    +        #         meta_element.build_dependencies.append(meta_dep)
    463 
    +        #     if dep.dep_type != 'build':
    464 
    +        #         meta_element.dependencies.append(meta_dep)
    459 465
    460 466 
             return meta_element
    461 467
    468 
    +    def _collect_elements(self, elements):
    469 
    +        elements_to_load = deque(elements)
    470 
    +
    471 
    +        def compare_unprocessed(dep_a, dep_b):
    472 
    +            if dep_a.dep_type != dep_b.dep_type:
    473 
    +                if dep_a.dep_type == Symbol.RUNTIME:
    474 
    +                    return 1
    475 
    +                if dep_b.dep_type == Symbol.RUNTIME:
    476 
    +                    return -1
    477 
    +
    478 
    +            element_a = dep_a.element
    479 
    +            element_b = dep_b.element
    480 
    +
    481 
    +            # All things being equal, string comparison.
    482 
    +            if element_a.name > element_b.name:
    483 
    +                return 1
    484 
    +            elif element_a.name < element_b.name:
    485 
    +                return -1
    486 
    +
    487 
    +            # Sort local elements before junction elements
    488 
    +            # and use string comparison between junction elements
    489 
    +            if element_a.junction and element_b.junction:
    490 
    +                if element_a.junction > element_b.junction:
    491 
    +                    return 1
    492 
    +                elif element_a.junction < element_b.junction:
    493 
    +                    return -1
    494 
    +            elif element_a.junction:
    495 
    +                return -1
    496 
    +            elif element_b.junction:
    497 
    +                return 1
    498 
    +
    499 
    +            return 0
    500 
    +
    501 
    +        print("#" * 60)
    502 
    +
    503 
    +        while elements_to_load:
    504 
    +            print([e.name for e in elements_to_load])
    505 
    +            element = elements_to_load.popleft()
    506 
    +
    507 
    +            print("Treating", element.name)
    508 
    +
    509 
    +            if element.visited:
    510 
    +                print("\tAlready visited")
    511 
    +                continue
    512 
    +
    513 
    +            unprocessed_dependencies = [
    514 
    +                dep
    515 
    +                for dep in element.dependencies
    516 
    +                if dep.element.visited is False
    517 
    +            ]
    518 
    +
    519 
    +            if unprocessed_dependencies:
    520 
    +                print("\tUnprocessed deps:", [e.element.name for e in unprocessed_dependencies])
    521 
    +                elements_to_load.appendleft(element)
    522 
    +                elements_to_load.extendleft(
    523 
    +                    dep.element
    524 
    +                    for dep in sorted(unprocessed_dependencies, key=cmp_to_key(compare_unprocessed), reverse=True)
    525 
    +                )
    526 
    +                continue
    527 
    +
    528 
    +            element.visited = True
    529 
    +            meta_element = element._loader._collect_element(element)
    530 
    +
    531 
    +            for dep in element.dependencies:
    532 
    +                print("\tElement has dependency:", dep.element.name)
    533 
    +                meta_dep = dep.element._loader._collect_element(dep.element)
    534 
    +
    535 
    +                if dep.dep_type != Symbol.RUNTIME:
    536 
    +                    meta_element.build_dependencies.append(meta_dep)
    537 
    +                if dep.dep_type != Symbol.BUILD:
    538 
    +                    meta_element.dependencies.append(meta_dep)
    539 
    +
    540 
    +            from operator import attrgetter
    541 
    +            meta_element.build_dependencies.sort(key=attrgetter("index"))
    542 
    +            meta_element.dependencies.sort(key=attrgetter("index"))
    543 
    +
    544 
    +        ret = [self._collect_element(e) for e in elements]
    545 
    +
    546 
    +        print("RETURN:", [e.name for e in ret])
    547 
    +        return ret
    548 
    +
    549 
    +        # levels = defaultdict(list)
    550 
    +
    551 
    +        # while elements_to_load:
    552 
    +        #     print("#" * 60)
    553 
    +        #     print([e.name for e in elements_to_load])
    554 
    +        #     element = elements_to_load.popleft()
    555 
    +        #     print("Treating element", element.name)
    556 
    +
    557 
    +        #     if element.level is not None:  # This node has already been treated
    558 
    +        #         print("Element", element.name, "already treated with level", element.level)
    559 
    +        #         continue
    560 
    +
    561 
    +        #     unprocessed_reverse_dependencies = [
    562 
    +        #         rdep
    563 
    +        #         for rdep in element.reverse_dependencies
    564 
    +        #         if rdep.level is None
    565 
    +        #     ]
    566 
    +
    567 
    +        #     if unprocessed_reverse_dependencies:
    568 
    +        #         print("Element has unprocessed reverse dependencies:", [e.name for e in unprocessed_reverse_dependencies])
    569 
    +        #         elements_to_load.appendleft(element)
    570 
    +        #         elements_to_load.extendleft(unprocessed_reverse_dependencies)
    571 
    +        #         continue
    572 
    +
    573 
    +        #     element.level = 1 + max((e.level for e in element.reverse_dependencies), default=0)
    574 
    +        #     levels[element.level].append(element)
    575 
    +        #     print("Element level is", element.level)
    576 
    +        #     print("Levels: ", {
    577 
    +        #         key: [e.name for e in value]
    578 
    +        #         for key, value in levels.items()
    579 
    +        #     })
    580 
    +
    581 
    +        #     def sort_by_dep_type(dep_a, dep_b):
    582 
    +        #         if dep_a.dep_type != dep_b.dep_type:
    583 
    +        #             if dep_a.dep_type == Symbol.RUNTIME:
    584 
    +        #                 return 1
    585 
    +        #             if dep_b.dep_type == Symbol.RUNTIME:
    586 
    +        #                 return -1
    587 
    +
    588 
    +        #         element_a = dep_a.element
    589 
    +        #         element_b = dep_b.element
    590 
    +
    591 
    +        #         # All things being equal, string comparison.
    592 
    +        #         if element_a.name > element_b.name:
    593 
    +        #             return 1
    594 
    +        #         elif element_a.name < element_b.name:
    595 
    +        #             return -1
    596 
    +
    597 
    +        #         # Sort local elements before junction elements
    598 
    +        #         # and use string comparison between junction elements
    599 
    +        #         if element_a.junction and element_b.junction:
    600 
    +        #             if element_a.junction > element_b.junction:
    601 
    +        #                 return 1
    602 
    +        #             elif element_a.junction < element_b.junction:
    603 
    +        #                 return -1
    604 
    +        #         elif element_a.junction:
    605 
    +        #             return -1
    606 
    +        #         elif element_b.junction:
    607 
    +        #             return 1
    608 
    +
    609 
    +        #         return 0
    610 
    +
    611 
    +        #     elements_to_load.extend(
    612 
    +        #         dep.element
    613 
    +        #         for dep in sorted(element.dependencies, key=cmp_to_key(sort_by_dep_type))
    614 
    +        #     )
    615 
    +
    616 
    +        # ret = []
    617 
    +
    618 
    +        # from operator import attrgetter
    619 
    +
    620 
    +        # print("Sorting levels")
    621 
    +
    622 
    +        # for _level in sorted(levels.keys(), reverse=True):
    623 
    +        #     print("## Treating level", _level)
    624 
    +        #     for element in levels[_level]:
    625 
    +        #         print("\t## Element", element.name)
    626 
    +        #         meta_element = self._collect_element(element)
    627 
    +
    628 
    +        #         for dep in element.dependencies:
    629 
    +        #             meta_dep = self._collect_element(dep.element)
    630 
    +
    631 
    +        #             if dep.dep_type != Symbol.RUNTIME:
    632 
    +        #                 meta_element.build_dependencies.append(meta_dep)
    633 
    +        #             if dep.dep_type != Symbol.BUILD:
    634 
    +        #                 meta_element.dependencies.append(meta_dep)
    635 
    +
    636 
    +        #         meta_element.build_dependencies.sort(key=attrgetter("index"))
    637 
    +        #         meta_element.dependencies.sort(key=attrgetter("index"))
    638 
    +
    639 
    +        #         if _level == 1:
    640 
    +        #             ret.append(meta_element)
    641 
    +
    642 
    +        # return ret
    643 
    +
    644 
    +
    462 645 
         # _get_loader():
    463 646 
         #
    464 647 
         # Return loader for specified junction

  • buildstream/_loader/metaelement.py
    ... ... @@ -17,9 +17,13 @@
    17 17 
     #  Authors:
    18 18 
     #        Tristan Van Berkom <tristan vanberkom codethink co uk>
    19 19
    20 
    +import itertools
    21 
    +
    20 22
    21 23 
     class MetaElement():
    22 24
    25 
    +    index_generator = itertools.count()
    26 
    +
    23 27 
         # MetaElement()
    24 28 
         #
    25 29 
         # An abstract object holding data suitable for constructing an Element
    ... ... @@ -55,3 +59,4 @@ class MetaElement():
    55 59 
             self.build_dependencies = []
    56 60 
             self.dependencies = []
    57 61 
             self.first_pass = first_pass
    62 
    +        self.index = next(self.index_generator)

  • buildstream/_profile.py
    ... ... @@ -43,7 +43,7 @@ initialized = False
    43 43 
     # The special 'all' value will enable all profiles.
    44 44 
     class Topics():
    45 45 
         CIRCULAR_CHECK = 'circ-dep-check'
    46 
    -    SORT_DEPENDENCIES = 'sort-deps'
    46 
    +    COLLECT_ELEMENTS = 'collect-elements'
    47 47 
         LOAD_LOADER = 'load-loader'
    48 48 
         LOAD_CONTEXT = 'load-context'
    49 49 
         LOAD_PROJECT = 'load-project'

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]