GNOME.org
 

[Notes] [Git][BuildStream/buildstream][tpollard/908] WIP: Artifact 'abstraction' class

Title: GitLab

Tom Pollard pushed to branch tpollard/908 at BuildStream / buildstream

Commits:

2 changed files:

Changes:

  • buildstream/_artifact.py
    1 
    +import os
    2 
    +import shutil
    3 
    +
    4 
    +from . import _yaml
    5 
    +from ._versions import BST_CORE_ARTIFACT_VERSION
    6 
    +from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \
    7 
    +    ErrorDomain, ArtifactElementError
    8 
    +from .utils import UtilError
    9 
    +from . import Scope
    10 
    +from . import utils
    11 
    +from .types import _KeyStrength
    12 
    +from .storage._casbaseddirectory import CasBasedDirectory
    13 
    +from .storage.directory import VirtualDirectoryError
    14 
    +
    15 
    +
    16 
    +# An Artifact class to abtract artifact operations
    17 
    +# from the Element class
    18 
    +#
    19 
    +# Args:
    20 
    +#     context (Context): The BuildStream context
    21 
    +#
    22 
    +class Artifact():
    23 
    +
    24 
    +    def __init__(self, context):
    25 
    +        self.__artifacts = context.artifactcache
    26 
    +        self.context = context
    27 
    +
    28 
    +    # Abstract the checking of buildtree state away from Element
    29 
    +    #
    30 
    +    def cached_buildtree(self, element):
    31 
    +
    32 
    +        context = self.context
    33 
    +
    34 
    +        if not element._cached():
    35 
    +            return False
    36 
    +
    37 
    +        key_strength = _KeyStrength.STRONG if context.get_strict() else _KeyStrength.WEAK
    38 
    +        if not self.__artifacts.contains_subdir_artifact(element, element._get_cache_key(strength=key_strength),
    39 
    +                                                         'buildtree'):
    40 
    +            return False
    41 
    +
    42 
    +        return True
    43 
    +
    44 
    +    def cache_artifact(self, element, rootdir, sandbox, collect, buildresult, keys, publicdata):
    45 
    +        if collect is not None:
    46 
    +            try:
    47 
    +                sandbox_vroot = sandbox.get_virtual_directory()
    48 
    +                collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
    49 
    +            except VirtualDirectoryError:
    50 
    +                # No collect directory existed
    51 
    +                collectvdir = None
    52 
    +
    53 
    +        context = self.context
    54 
    +
    55 
    +        assemblevdir = CasBasedDirectory(cas_cache=self.__artifacts.cas, ref=None)
    56 
    +        logsvdir = assemblevdir.descend("logs", create=True)
    57 
    +        metavdir = assemblevdir.descend("meta", create=True)
    58 
    +        buildtreevdir = assemblevdir.descend("buildtree", create=True)
    59 
    +
    60 
    +        # Create artifact directory structure
    61 
    +        assembledir = os.path.join(rootdir, 'artifact')
    62 
    +        logsdir = os.path.join(assembledir, 'logs')
    63 
    +        metadir = os.path.join(assembledir, 'meta')
    64 
    +        os.mkdir(assembledir)
    65 
    +        os.mkdir(logsdir)
    66 
    +        os.mkdir(metadir)
    67 
    +
    68 
    +        if collect is not None and collectvdir is not None:
    69 
    +            filesvdir = assemblevdir.descend("files", create=True)
    70 
    +            filesvdir.import_files(collectvdir)
    71 
    +
    72 
    +        cache_buildtrees = context.cache_buildtrees
    73 
    +        build_success = buildresult[0]
    74 
    +
    75 
    +        # cache_buildtrees defaults to 'always', as such the
    76 
    +        # default behaviour is to attempt to cache them. If only
    77 
    +        # caching failed artifact buildtrees, then query the build
    78 
    +        # result. Element types without a build-root dir will be cached
    79 
    +        # with an empty buildtreedir regardless of this configuration.
    80 
    +
    81 
    +        if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):
    82 
    +            sandbox_vroot = sandbox.get_virtual_directory()
    83 
    +            try:
    84 
    +                sandbox_build_dir = sandbox_vroot.descend(
    85 
    +                    element.get_variable('build-root').lstrip(os.sep).split(os.sep))
    86 
    +                buildtreevdir.import_files(sandbox_build_dir)
    87 
    +            except VirtualDirectoryError:
    88 
    +                # Directory could not be found. Pre-virtual
    89 
    +                # directory behaviour was to continue silently
    90 
    +                # if the directory could not be found.
    91 
    +                pass
    92 
    +
    93 
    +        # Write some logs out to normal directories: logsdir and metadir
    94 
    +        # Copy build log
    95 
    +        log_filename = context.get_log_filename()
    96 
    +        element._build_log_path = os.path.join(logsdir, 'build.log')
    97 
    +        if log_filename:
    98 
    +            shutil.copyfile(log_filename, element._build_log_path)
    99 
    +
    100 
    +        # Store public data
    101 
    +        _yaml.dump(_yaml.node_sanitize(publicdata), os.path.join(metadir, 'public.yaml'))
    102 
    +
    103 
    +        # Store result
    104 
    +        build_result_dict = {"success": buildresult[0], "description": buildresult[1]}
    105 
    +        if buildresult[2] is not None:
    106 
    +            build_result_dict["detail"] = buildresult[2]
    107 
    +        _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
    108 
    +
    109 
    +        # ensure we have cache keys
    110 
    +        element._assemble_done()
    111 
    +
    112 
    +        # Store keys.yaml
    113 
    +        _yaml.dump(_yaml.node_sanitize({
    114 
    +            'strong': element._get_cache_key(),
    115 
    +            'weak': element._get_cache_key(_KeyStrength.WEAK),
    116 
    +        }), os.path.join(metadir, 'keys.yaml'))
    117 
    +
    118 
    +        # Store dependencies.yaml
    119 
    +        _yaml.dump(_yaml.node_sanitize({
    120 
    +            e.name: e._get_cache_key() for e in element.dependencies(Scope.BUILD)
    121 
    +        }), os.path.join(metadir, 'dependencies.yaml'))
    122 
    +
    123 
    +        # Store workspaced.yaml
    124 
    +        _yaml.dump(_yaml.node_sanitize({
    125 
    +            'workspaced': bool(element._get_workspace())
    126 
    +        }), os.path.join(metadir, 'workspaced.yaml'))
    127 
    +
    128 
    +        # Store workspaced-dependencies.yaml
    129 
    +        _yaml.dump(_yaml.node_sanitize({
    130 
    +            'workspaced-dependencies': [
    131 
    +                e.name for e in element.dependencies(Scope.BUILD)
    132 
    +                if e._get_workspace()
    133 
    +            ]
    134 
    +        }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
    135 
    +
    136 
    +        metavdir.import_files(metadir)
    137 
    +        logsvdir.import_files(logsdir)
    138 
    +
    139 
    +        artifact_size = assemblevdir.get_size()
    140 
    +        self.__artifacts.commit(element, assemblevdir, keys)
    141 
    +
    142 
    +        if collect is not None and collectvdir is None:
    143 
    +            raise ArtifactElementError("Not Found")
    144 
    +
    145 
    +        return artifact_size
    146 
    +
    147 
    +    # load_public_data():
    148 
    +    #
    149 
    +    # Loads the public data from the cached artifact
    150 
    +    #
    151 
    +    def load_public_data(self, element):
    152 
    +
    153 
    +        assert element._cached()
    154 
    +
    155 
    +        # Load the public data from the artifact
    156 
    +        artifact_base, _ = self.extract(element)
    157 
    +        metadir = os.path.join(artifact_base, 'meta')
    158 
    +        data = _yaml.load(os.path.join(metadir, 'public.yaml'))
    159 
    +
    160 
    +        return data
    161 
    +
    162 
    +    def load_build_result(self, element, key):
    163 
    +
    164 
    +        assert key is not None
    165 
    +        artifact_base, _ = self.extract(element, key)
    166 
    +
    167 
    +        metadir = os.path.join(artifact_base, 'meta')
    168 
    +        result_path = os.path.join(metadir, 'build-result.yaml')
    169 
    +        if not os.path.exists(result_path):
    170 
    +            build_result = (True, "succeeded", None)
    171 
    +            return build_result
    172 
    +
    173 
    +        data = _yaml.load(result_path)
    174 
    +        build_result = (data["success"], data.get("description"), data.get("detail"))
    175 
    +
    176 
    +        return build_result
    177 
    +
    178 
    +    def extract(self, element, key=None):
    179 
    +
    180 
    +        if key is None:
    181 
    +            context = self.context
    182 
    +
    183 
    +            # Use weak cache key, if context allows use of weak cache keys
    184 
    +            key_strength = _KeyStrength.STRONG
    185 
    +            key = element._get_cache_key(strength=key_strength)
    186 
    +            if not context.get_strict() and not key:
    187 
    +                key = element._get_cache_key(strength=_KeyStrength.WEAK)
    188 
    +
    189 
    +        return (self.__artifacts.extract(element, key), key)
    190 
    +
    191 
    +    def get_artifact_metadata_keys(self, element, key, metadata_keys):
    192 
    +
    193 
    +        # Now extract it and possibly derive the key
    194 
    +        artifact_base, key = self.extract(element, key)
    195 
    +
    196 
    +        # Now try the cache, once we're sure about the key
    197 
    +        if key in metadata_keys:
    198 
    +            return (metadata_keys[key]['strong'],
    199 
    +                    metadata_keys[key]['weak'], None)
    200 
    +
    201 
    +        # Parse the expensive yaml now and cache the result
    202 
    +        meta_file = os.path.join(artifact_base, 'meta', 'keys.yaml')
    203 
    +        meta = _yaml.load(meta_file)
    204 
    +        strong_key = meta['strong']
    205 
    +        weak_key = meta['weak']
    206 
    +
    207 
    +        assert key in (strong_key, weak_key)
    208 
    +
    209 
    +        metadata_keys[strong_key] = meta
    210 
    +        metadata_keys[weak_key] = meta
    211 
    +
    212 
    +        return (strong_key, weak_key, metadata_keys)
    213 
    +
    214 
    +    def get_artifact_metadata_dependencies(self, element, key, metadata_dependencies, metadata_keys):
    215 
    +
    216 
    +        # Extract it and possibly derive the key
    217 
    +        artifact_base, key = self.extract(element, key)
    218 
    +
    219 
    +        # Now try the cache, once we're sure about the key
    220 
    +        if key in metadata_dependencies:
    221 
    +            return (metadata_dependencies[key], None, None)
    222 
    +
    223 
    +        # Parse the expensive yaml now and cache the result
    224 
    +        meta_file = os.path.join(artifact_base, 'meta', 'dependencies.yaml')
    225 
    +        meta = _yaml.load(meta_file)
    226 
    +
    227 
    +        # Cache it under both strong and weak keys
    228 
    +        strong_key, weak_key, metadata_keys = self.get_artifact_metadata_keys(element, key, metadata_keys)
    229 
    +        metadata_dependencies[strong_key] = meta
    230 
    +        metadata_dependencies[weak_key] = meta
    231 
    +
    232 
    +        return (meta, metadata_dependencies, metadata_keys)
    233 
    +
    234 
    +    def get_artifact_metadata_workspaced(self, element, key, metadata_workspaced, metadata_keys):
    235 
    +
    236 
    +        # Extract it and possibly derive the key
    237 
    +        artifact_base, key = self.extract(element, key)
    238 
    +
    239 
    +        # Now try the cache, once we're sure about the key
    240 
    +        if key in metadata_workspaced:
    241 
    +            return (metadata_workspaced[key], None, None)
    242 
    +
    243 
    +        # Parse the expensive yaml now and cache the result
    244 
    +        meta_file = os.path.join(artifact_base, 'meta', 'workspaced.yaml')
    245 
    +        meta = _yaml.load(meta_file)
    246 
    +        workspaced = meta['workspaced']
    247 
    +
    248 
    +        # Cache it under both strong and weak keys
    249 
    +        strong_key, weak_key, metadata_keys = self.get_artifact_metadata_keys(element, key, metadata_keys)
    250 
    +        metadata_workspaced[strong_key] = workspaced
    251 
    +        metadata_workspaced[weak_key] = workspaced
    252 
    +
    253 
    +        return (workspaced, metadata_workspaced, metadata_keys)
    254 
    +
    255 
    +    def get_artifact_metadata_workspaced_dependencies(self, element, key, metadata_workspaced_dependencies,
    256 
    +                                                      metadata_keys):
    257 
    +
    258 
    +        # Extract it and possibly derive the key
    259 
    +        artifact_base, key = self.extract(element, key)
    260 
    +
    261 
    +        # Now try the cache, once we're sure about the key
    262 
    +        if key in metadata_workspaced_dependencies:
    263 
    +            return (metadata_workspaced_dependencies[key], None, None)
    264 
    +
    265 
    +        # Parse the expensive yaml now and cache the result
    266 
    +        meta_file = os.path.join(artifact_base, 'meta', 'workspaced-dependencies.yaml')
    267 
    +        meta = _yaml.load(meta_file)
    268 
    +        workspaced = meta['workspaced-dependencies']
    269 
    +
    270 
    +        # Cache it under both strong and weak keys
    271 
    +        strong_key, weak_key, metadata_keys = self.get_artifact_metadata_keys(element, key, metadata_keys)
    272 
    +        metadata_workspaced_dependencies[strong_key] = workspaced
    273 
    +        metadata_workspaced_dependencies[weak_key] = workspaced
    274 
    +        return (workspaced, metadata_workspaced_dependencies, metadata_keys)

  • buildstream/element.py
    ... ... @@ -88,7 +88,7 @@ from . import _yaml
    88 88 
     from ._variables import Variables
    89 89 
     from ._versions import BST_CORE_ARTIFACT_VERSION
    90 90 
     from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \
    91 
    -    ErrorDomain
    91 
    +    ErrorDomain, ArtifactElementError
    92 92 
     from .utils import UtilError
    93 93 
     from . import Plugin, Consistency, Scope
    94 94 
     from . import SandboxFlags, SandboxCommandError
    ... ... @@ -100,6 +100,7 @@ from ._platform import Platform
    100 100 
     from .sandbox._config import SandboxConfig
    101 101 
     from .sandbox._sandboxremote import SandboxRemote
    102 102 
     from .types import _KeyStrength, CoreWarnings
    103 
    +from ._artifact import Artifact
    103 104
    104 105 
     from .storage.directory import Directory
    105 106 
     from .storage._filebaseddirectory import FileBasedDirectory
    ... ... @@ -218,6 +219,7 @@ class Element(Plugin):
    218 219 
             self.__required = False                 # Whether the artifact is required in the current session
    219 220 
             self.__build_result = None              # The result of assembling this Element (success, description, detail)
    220 221 
             self._build_log_path = None            # The path of the build log for this Element
    222 
    +        self.__artifact = Artifact(context)
    221 223
    222 224 
             self.__batch_prepare_assemble = False         # Whether batching across prepare()/assemble() is configured
    223 225 
             self.__batch_prepare_assemble_flags = 0       # Sandbox flags for batching across prepare()/assemble()
    ... ... @@ -1671,112 +1673,18 @@ class Element(Plugin):
    1671 1673 
                         cleanup_rootdir()
    1672 1674
    1673 1675 
         def _cache_artifact(self, rootdir, sandbox, collect):
    1676 
    +        buildresult = self.__build_result
    1677 
    +        keys = self.__get_cache_keys_for_commit()
    1678 
    +        publicdata = self.__dynamic_public
    1674 1679 
             with self.timed_activity("Caching artifact"):
    1675 
    -            if collect is not None:
    1676 
    -                try:
    1677 
    -                    sandbox_vroot = sandbox.get_virtual_directory()
    1678 
    -                    collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
    1679 
    -                except VirtualDirectoryError:
    1680 
    -                    # No collect directory existed
    1681 
    -                    collectvdir = None
    1682 
    -
    1683 
    -            context = self._get_context()
    1684 
    -
    1685 
    -            assemblevdir = CasBasedDirectory(cas_cache=context.artifactcache.cas, ref=None)
    1686 
    -            logsvdir = assemblevdir.descend("logs", create=True)
    1687 
    -            metavdir = assemblevdir.descend("meta", create=True)
    1688 
    -            buildtreevdir = assemblevdir.descend("buildtree", create=True)
    1689 
    -
    1690 
    -            # Create artifact directory structure
    1691 
    -            assembledir = os.path.join(rootdir, 'artifact')
    1692 
    -            logsdir = os.path.join(assembledir, 'logs')
    1693 
    -            metadir = os.path.join(assembledir, 'meta')
    1694 
    -            os.mkdir(assembledir)
    1695 
    -            os.mkdir(logsdir)
    1696 
    -            os.mkdir(metadir)
    1697 
    -
    1698 
    -            if collect is not None and collectvdir is not None:
    1699 
    -                filesvdir = assemblevdir.descend("files", create=True)
    1700 
    -                filesvdir.import_files(collectvdir)
    1701 
    -
    1702 
    -            cache_buildtrees = context.cache_buildtrees
    1703 
    -            build_success = self.__build_result[0]
    1704 
    -
    1705 
    -            # cache_buildtrees defaults to 'always', as such the
    1706 
    -            # default behaviour is to attempt to cache them. If only
    1707 
    -            # caching failed artifact buildtrees, then query the build
    1708 
    -            # result. Element types without a build-root dir will be cached
    1709 
    -            # with an empty buildtreedir regardless of this configuration.
    1710 
    -
    1711 
    -            if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):
    1712 
    -                sandbox_vroot = sandbox.get_virtual_directory()
    1713 
    -                try:
    1714 
    -                    sandbox_build_dir = sandbox_vroot.descend(
    1715 
    -                        self.get_variable('build-root').lstrip(os.sep).split(os.sep))
    1716 
    -                    buildtreevdir.import_files(sandbox_build_dir)
    1717 
    -                except VirtualDirectoryError:
    1718 
    -                    # Directory could not be found. Pre-virtual
    1719 
    -                    # directory behaviour was to continue silently
    1720 
    -                    # if the directory could not be found.
    1721 
    -                    pass
    1722 
    -
    1723 
    -            # Write some logs out to normal directories: logsdir and metadir
    1724 
    -            # Copy build log
    1725 
    -            log_filename = context.get_log_filename()
    1726 
    -            self._build_log_path = os.path.join(logsdir, 'build.log')
    1727 
    -            if log_filename:
    1728 
    -                shutil.copyfile(log_filename, self._build_log_path)
    1729 
    -
    1730 
    -            # Store public data
    1731 
    -            _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
    1732 
    -
    1733 
    -            # Store result
    1734 
    -            build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
    1735 
    -            if self.__build_result[2] is not None:
    1736 
    -                build_result_dict["detail"] = self.__build_result[2]
    1737 
    -            _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
    1738 
    -
    1739 
    -            # ensure we have cache keys
    1740 
    -            self._assemble_done()
    1741 
    -
    1742 
    -            # Store keys.yaml
    1743 
    -            _yaml.dump(_yaml.node_sanitize({
    1744 
    -                'strong': self._get_cache_key(),
    1745 
    -                'weak': self._get_cache_key(_KeyStrength.WEAK),
    1746 
    -            }), os.path.join(metadir, 'keys.yaml'))
    1747 
    -
    1748 
    -            # Store dependencies.yaml
    1749 
    -            _yaml.dump(_yaml.node_sanitize({
    1750 
    -                e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
    1751 
    -            }), os.path.join(metadir, 'dependencies.yaml'))
    1752 
    -
    1753 
    -            # Store workspaced.yaml
    1754 
    -            _yaml.dump(_yaml.node_sanitize({
    1755 
    -                'workspaced': bool(self._get_workspace())
    1756 
    -            }), os.path.join(metadir, 'workspaced.yaml'))
    1757 
    -
    1758 
    -            # Store workspaced-dependencies.yaml
    1759 
    -            _yaml.dump(_yaml.node_sanitize({
    1760 
    -                'workspaced-dependencies': [
    1761 
    -                    e.name for e in self.dependencies(Scope.BUILD)
    1762 
    -                    if e._get_workspace()
    1763 
    -                ]
    1764 
    -            }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
    1765 
    -
    1766 
    -            metavdir.import_files(metadir)
    1767 
    -            logsvdir.import_files(logsdir)
    1768 
    -
    1769 
    -            artifact_size = assemblevdir.get_size()
    1770 
    -            self.__artifacts.commit(self, assemblevdir, self.__get_cache_keys_for_commit())
    1771 
    -
    1772 
    -            if collect is not None and collectvdir is None:
    1680 
    +            try:
    1681 
    +                return self.__artifact.cache_artifact(self, rootdir, sandbox, collect, buildresult, keys, publicdata)
    1682 
    +            except ArtifactElementError:
    1773 1683 
                     raise ElementError(
    1774 1684 
                         "Directory '{}' was not found inside the sandbox, "
    1775 1685 
                         "unable to collect artifact contents"
    1776 1686 
                         .format(collect))
    1777 1687
    1778 
    -        return artifact_size
    1779 
    -
    1780 1688 
         def _get_build_log(self):
    1781 1689 
             return self._build_log_path
    1782 1690
    ... ... @@ -2071,17 +1979,7 @@ class Element(Plugin):
    2071 1979 
         #             not its contents.
    2072 1980 
         #
    2073 1981 
         def _cached_buildtree(self):
    2074 
    -        context = self._get_context()
    2075 
    -
    2076 
    -        if not self._cached():
    2077 
    -            return False
    2078 
    -
    2079 
    -        key_strength = _KeyStrength.STRONG if context.get_strict() else _KeyStrength.WEAK
    2080 
    -        if not self.__artifacts.contains_subdir_artifact(self, self._get_cache_key(strength=key_strength),
    2081 
    -                                                         'buildtree'):
    2082 
    -            return False
    2083 
    -
    2084 
    -        return True
    1982 
    +        return self.__artifact.cached_buildtree(self)
    2085 1983
    2086 1984 
         # _fetch()
    2087 1985 
         #
    ... ... @@ -2637,16 +2535,9 @@ class Element(Plugin):
    2637 2535 
         #
    2638 2536 
         def __extract(self, key=None):
    2639 2537
    2640 
    -        if key is None:
    2641 
    -            context = self._get_context()
    2642 
    -            key = self.__strict_cache_key
    2643 
    -
    2644 
    -            # Use weak cache key, if artifact is missing for strong cache key
    2645 
    -            # and the context allows use of weak cache keys
    2646 
    -            if not context.get_strict() and not self.__artifacts.contains(self, key):
    2647 
    -                key = self._get_cache_key(strength=_KeyStrength.WEAK)
    2538 
    +        artifact_base, key = self.__artifact.extract(self, key)
    2648 2539
    2649 
    -        return (self.__artifacts.extract(self, key), key)
    2540 
    +        return (artifact_base, key)
    2650 2541
    2651 2542 
         # __get_artifact_metadata_keys():
    2652 2543 
         #
    ... ... @@ -2661,24 +2552,14 @@ class Element(Plugin):
    2661 2552 
         #
    2662 2553 
         def __get_artifact_metadata_keys(self, key=None):
    2663 2554
    2664 
    -        # Now extract it and possibly derive the key
    2665 
    -        artifact_base, key = self.__extract(key)
    2555 
    +        metadata_keys = self.__metadata_keys
    2666 2556
    2667 
    -        # Now try the cache, once we're sure about the key
    2668 
    -        if key in self.__metadata_keys:
    2669 
    -            return (self.__metadata_keys[key]['strong'],
    2670 
    -                    self.__metadata_keys[key]['weak'])
    2557 
    +        strong_key, weak_key, metadata_keys = self.__artifact.get_artifact_metadata_keys(self, key, metadata_keys)
    2671 2558
    2672 
    -        # Parse the expensive yaml now and cache the result
    2673 
    -        meta_file = os.path.join(artifact_base, 'meta', 'keys.yaml')
    2674 
    -        meta = _yaml.load(meta_file)
    2675 
    -        strong_key = meta['strong']
    2676 
    -        weak_key = meta['weak']
    2559 
    +        # Update keys if needed
    2560 
    +        if metadata_keys:
    2561 
    +            self.__metadata_keys = metadata_keys
    2677 2562
    2678 
    -        assert key in (strong_key, weak_key)
    2679 
    -
    2680 
    -        self.__metadata_keys[strong_key] = meta
    2681 
    -        self.__metadata_keys[weak_key] = meta
    2682 2563 
             return (strong_key, weak_key)
    2683 2564
    2684 2565 
         # __get_artifact_metadata_dependencies():
    ... ... @@ -2693,21 +2574,16 @@ class Element(Plugin):
    2693 2574 
         #
    2694 2575 
         def __get_artifact_metadata_dependencies(self, key=None):
    2695 2576
    2696 
    -        # Extract it and possibly derive the key
    2697 
    -        artifact_base, key = self.__extract(key)
    2698 
    -
    2699 
    -        # Now try the cache, once we're sure about the key
    2700 
    -        if key in self.__metadata_dependencies:
    2701 
    -            return self.__metadata_dependencies[key]
    2577 
    +        metadata = [self.__metadata_dependencies, self.__metadata_keys]
    2578 
    +        meta, meta_deps, meta_keys = self.__artifact.get_artifact_metadata_dependencies(self, key, *metadata)
    2702 2579
    2703 
    -        # Parse the expensive yaml now and cache the result
    2704 
    -        meta_file = os.path.join(artifact_base, 'meta', 'dependencies.yaml')
    2705 
    -        meta = _yaml.load(meta_file)
    2580 
    +        # Update deps if needed
    2581 
    +        if meta_deps:
    2582 
    +            self.__metadata_dependencies = meta_deps
    2583 
    +            # Update keys if needed, no need to check if deps not updated
    2584 
    +            if meta_keys:
    2585 
    +                self.__metadata_keys = meta_keys
    2706 2586
    2707 
    -        # Cache it under both strong and weak keys
    2708 
    -        strong_key, weak_key = self.__get_artifact_metadata_keys(key)
    2709 
    -        self.__metadata_dependencies[strong_key] = meta
    2710 
    -        self.__metadata_dependencies[weak_key] = meta
    2711 2587 
             return meta
    2712 2588
    2713 2589 
         # __get_artifact_metadata_workspaced():
    ... ... @@ -2720,24 +2596,19 @@ class Element(Plugin):
    2720 2596 
         # Returns:
    2721 2597 
         #     (bool): Whether the given artifact was workspaced
    2722 2598 
         #
    2723 
    -    def __get_artifact_metadata_workspaced(self, key=None):
    2724 2599
    2725 
    -        # Extract it and possibly derive the key
    2726 
    -        artifact_base, key = self.__extract(key)
    2600 
    +    def __get_artifact_metadata_workspaced(self, key=None):
    2727 2601
    2728 
    -        # Now try the cache, once we're sure about the key
    2729 
    -        if key in self.__metadata_workspaced:
    2730 
    -            return self.__metadata_workspaced[key]
    2602 
    +        metadata = [self.__metadata_workspaced, self.__metadata_keys]
    2603 
    +        workspaced, meta_workspaced, meta_keys = self.__artifact.get_artifact_metadata_workspaced(self, key, *metadata)
    2731 2604
    2732 
    -        # Parse the expensive yaml now and cache the result
    2733 
    -        meta_file = os.path.join(artifact_base, 'meta', 'workspaced.yaml')
    2734 
    -        meta = _yaml.load(meta_file)
    2735 
    -        workspaced = meta['workspaced']
    2605 
    +        # Update workspaced if needed
    2606 
    +        if meta_workspaced:
    2607 
    +            self.__metadata_workspaced = meta_workspaced
    2608 
    +            # Update keys if needed, no need to check if workspaced not updated
    2609 
    +            if meta_keys:
    2610 
    +                self.__metadata_keys = meta_keys
    2736 2611
    2737 
    -        # Cache it under both strong and weak keys
    2738 
    -        strong_key, weak_key = self.__get_artifact_metadata_keys(key)
    2739 
    -        self.__metadata_workspaced[strong_key] = workspaced
    2740 
    -        self.__metadata_workspaced[weak_key] = workspaced
    2741 2612 
             return workspaced
    2742 2613
    2743 2614 
         # __get_artifact_metadata_workspaced_dependencies():
    ... ... @@ -2752,22 +2623,17 @@ class Element(Plugin):
    2752 2623 
         #
    2753 2624 
         def __get_artifact_metadata_workspaced_dependencies(self, key=None):
    2754 2625
    2755 
    -        # Extract it and possibly derive the key
    2756 
    -        artifact_base, key = self.__extract(key)
    2626 
    +        metadata = [self.__metadata_workspaced_dependencies, self.__metadata_keys]
    2627 
    +        workspaced, meta_workspaced_deps,\
    2628 
    +           meta_keys = self.__artifact.get_artifact_metadata_workspaced_dependencies(self, key, *metadata)
    2757 2629
    2758 
    -        # Now try the cache, once we're sure about the key
    2759 
    -        if key in self.__metadata_workspaced_dependencies:
    2760 
    -            return self.__metadata_workspaced_dependencies[key]
    2630 
    +        # Update workspaced if needed
    2631 
    +        if meta_workspaced_deps:
    2632 
    +            self.__metadata_workspaced_dependencies = meta_workspaced_deps
    2633 
    +            # Update keys if needed, no need to check if workspaced not updated
    2634 
    +            if meta_keys:
    2635 
    +                self.__metadata_keys = meta_keys
    2761 2636
    2762 
    -        # Parse the expensive yaml now and cache the result
    2763 
    -        meta_file = os.path.join(artifact_base, 'meta', 'workspaced-dependencies.yaml')
    2764 
    -        meta = _yaml.load(meta_file)
    2765 
    -        workspaced = meta['workspaced-dependencies']
    2766 
    -
    2767 
    -        # Cache it under both strong and weak keys
    2768 
    -        strong_key, weak_key = self.__get_artifact_metadata_keys(key)
    2769 
    -        self.__metadata_workspaced_dependencies[strong_key] = workspaced
    2770 
    -        self.__metadata_workspaced_dependencies[weak_key] = workspaced
    2771 2637 
             return workspaced
    2772 2638
    2773 2639 
         # __load_public_data():
    ... ... @@ -2775,29 +2641,20 @@ class Element(Plugin):
    2775 2641 
         # Loads the public data from the cached artifact
    2776 2642 
         #
    2777 2643 
         def __load_public_data(self):
    2778 
    -        self.__assert_cached()
    2779 2644 
             assert self.__dynamic_public is None
    2780 2645
    2781 
    -        # Load the public data from the artifact
    2782 
    -        artifact_base, _ = self.__extract()
    2783 
    -        metadir = os.path.join(artifact_base, 'meta')
    2784 
    -        self.__dynamic_public = _yaml.load(os.path.join(metadir, 'public.yaml'))
    2646 
    +        self.__dynamic_public = self.__artifact.load_public_data(self)
    2785 2647
    2786 2648 
         def __load_build_result(self, keystrength):
    2787 2649 
             self.__assert_cached(keystrength=keystrength)
    2788 2650 
             assert self.__build_result is None
    2789 2651
    2790 
    -        artifact_base, _ = self.__extract(key=self.__weak_cache_key if keystrength is _KeyStrength.WEAK
    2791 
    -                                          else self.__strict_cache_key)
    2792 
    -
    2793 
    -        metadir = os.path.join(artifact_base, 'meta')
    2794 
    -        result_path = os.path.join(metadir, 'build-result.yaml')
    2795 
    -        if not os.path.exists(result_path):
    2796 
    -            self.__build_result = (True, "succeeded", None)
    2797 
    -            return
    2652 
    +        # _get_cache_key with _KeyStrength.STRONG returns self.__cache_key, which can be `None`
    2653 
    +        # leading to a failed assertion from extract() using get_artifact_name(), so explicility pass
    2654 
    +        # self.__strict_cache_key
    2655 
    +        key = self.__weak_cache_key if keystrength is _KeyStrength.WEAK else self.__strict_cache_key
    2798 2656
    2799 
    -        data = _yaml.load(result_path)
    2800 
    -        self.__build_result = (data["success"], data.get("description"), data.get("detail"))
    2657 
    +        self.__build_result = self.__artifact.load_build_result(self, key)
    2801 2658
    2802 2659 
         def __get_build_result(self, keystrength):
    2803 2660 
             if keystrength is None:

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]