GNOME.org
 

[Notes] [Git][BuildStream/buildstream][jmac/remote_execution_client] 9 commits: _casbaseddirectory.py: add _save() function.

Title: GitLab

Martin Blanchard pushed to branch jmac/remote_execution_client at BuildStream / buildstream

Commits:

10 changed files:

Changes:

  • buildstream/_project.py
    ... ... @@ -129,6 +129,7 @@ class Project():
    129 129
    130 130 
             self.artifact_cache_specs = None
    131 131 
             self._sandbox = None
    132 
    +        self._remote_execution = None
    132 133 
             self._splits = None
    133 134
    134 135 
             self._context.add_project(self)
    ... ... @@ -460,7 +461,7 @@ class Project():
    460 461 
                 'aliases', 'name',
    461 462 
                 'artifacts', 'options',
    462 463 
                 'fail-on-overlap', 'shell', 'fatal-warnings',
    463 
    -            'ref-storage', 'sandbox', 'mirrors'
    464 
    +            'ref-storage', 'sandbox', 'mirrors', 'remote-execution'
    464 465 
             ])
    465 466
    466 467 
             #
    ... ... @@ -478,6 +479,9 @@ class Project():
    478 479 
             # Load sandbox configuration
    479 480 
             self._sandbox = _yaml.node_get(config, Mapping, 'sandbox')
    480 481
    482 
    +        # Load remote execution configuration
    483 
    +        self._remote_execution = _yaml.node_get(config, Mapping, 'remote-execution')
    484 
    +
    481 485 
             # Load project split rules
    482 486 
             self._splits = _yaml.node_get(config, Mapping, 'split-rules')
    483 487

  • buildstream/buildelement.py
    ... ... @@ -155,6 +155,9 @@ class BuildElement(Element):
    155 155 
                 command_dir = build_root
    156 156 
             sandbox.set_work_directory(command_dir)
    157 157
    158 
    +        # Tell sandbox which directory is preserved in the finished artifact
    159 
    +        sandbox.set_output_directory(install_root)
    160 
    +
    158 161 
             # Setup environment
    159 162 
             sandbox.set_environment(self.get_environment())
    160 163

  • buildstream/data/projectconfig.yaml
    ... ... @@ -204,3 +204,6 @@ shell:
    204 204 
       # Command to run when `bst shell` does not provide a command
    205 205 
       #
    206 206 
       command: [ 'sh', '-i' ]
    207 
    +
    208 
    +remote-execution:
    209 
    +  url: ""
    \ No newline at end of file

  • buildstream/element.py
    ... ... @@ -95,6 +95,7 @@ from . import _site
    95 95 
     from ._platform import Platform
    96 96 
     from .plugin import CoreWarnings
    97 97 
     from .sandbox._config import SandboxConfig
    98 
    +from .sandbox._sandboxremote import SandboxRemote
    98 99
    99 100 
     from .storage.directory import Directory
    100 101 
     from .storage._filebaseddirectory import FileBasedDirectory
    ... ... @@ -250,6 +251,9 @@ class Element(Plugin):
    250 251 
             # Extract Sandbox config
    251 252 
             self.__sandbox_config = self.__extract_sandbox_config(meta)
    252 253
    254 
    +        # Extract remote execution URL
    255 
    +        self.__remote_execution_url = self.__extract_remote_execution_config(meta)
    256 
    +
    253 257 
         def __lt__(self, other):
    254 258 
             return self.name < other.name
    255 259
    ... ... @@ -1570,6 +1574,8 @@ class Element(Plugin):
    1570 1574 
                     finally:
    1571 1575 
                         if collect is not None:
    1572 1576 
                             try:
    1577 
    +                            # Sandbox will probably have replaced its virtual directory, so get it again
    1578 
    +                            sandbox_vroot = sandbox.get_virtual_directory()
    1573 1579 
                                 collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
    1574 1580 
                             except VirtualDirectoryError:
    1575 1581 
                                 # No collect directory existed
    ... ... @@ -2146,7 +2152,32 @@ class Element(Plugin):
    2146 2152 
             project = self._get_project()
    2147 2153 
             platform = Platform.get_platform()
    2148 2154
    2149 
    -        if directory is not None and os.path.exists(directory):
    2155 
    +        if self.__remote_execution_url and self.BST_VIRTUAL_DIRECTORY:
    2156 
    +            if not self.__artifacts.has_push_remotes(element=self):
    2157 
    +                # Give an early warning if remote execution will not work
    2158 
    +                raise ElementError("Artifact {} is configured to use remote execution but has no push remotes. "
    2159 
    +                                   .format(self.name) +
    2160 
    +                                   "The remote artifact server(s) may not be correctly configured or contactable.")
    2161 
    +
    2162 
    +            self.info("Using a remote sandbox for artifact {}".format(self.name))
    2163 
    +
    2164 
    +            sandbox = SandboxRemote(context, project,
    2165 
    +                                    directory,
    2166 
    +                                    stdout=stdout,
    2167 
    +                                    stderr=stderr,
    2168 
    +                                    config=config,
    2169 
    +                                    server_url=self.__remote_execution_url,
    2170 
    +                                    allow_real_directory=False)
    2171 
    +            yield sandbox
    2172 
    +
    2173 
    +        elif directory is not None and os.path.exists(directory):
    2174 
    +            if self.__remote_execution_url:
    2175 
    +                self.warn("Artifact {} is configured to use remote execution but element plugin does not support it."
    2176 
    +                          .format(self.name), detail="Element plugin '{kind}' does not support virtual directories."
    2177 
    +                          .format(kind=self.get_kind()), warning_token="remote-failure")
    2178 
    +
    2179 
    +                self.info("Falling back to local sandbox for artifact {}".format(self.name))
    2180 
    +
    2150 2181 
                 sandbox = platform.create_sandbox(context, project,
    2151 2182 
                                                   directory,
    2152 2183 
                                                   stdout=stdout,
    ... ... @@ -2318,6 +2349,20 @@ class Element(Plugin):
    2318 2349 
             return SandboxConfig(self.node_get_member(sandbox_config, int, 'build-uid'),
    2319 2350 
                                  self.node_get_member(sandbox_config, int, 'build-gid'))
    2320 2351
    2352 
    +    # Remote execution configuration data (server URL), to be used by the remote sandbox.
    2353 
    +    #
    2354 
    +    def __extract_remote_execution_config(self, meta):
    2355 
    +        if self.__is_junction:
    2356 
    +            return None
    2357 
    +        else:
    2358 
    +            project = self._get_project()
    2359 
    +            project.ensure_fully_loaded()
    2360 
    +            if project._remote_execution:
    2361 
    +                rexec_config = _yaml.node_chain_copy(project._remote_execution)
    2362 
    +                return self.node_get_member(rexec_config, str, 'url')
    2363 
    +            else:
    2364 
    +                return None
    2365 
    +
    2321 2366 
         # This makes a special exception for the split rules, which
    2322 2367 
         # elements may extend but whos defaults are defined in the project.
    2323 2368 
         #

  • buildstream/plugins/elements/autotools.py
    ... ... @@ -57,7 +57,7 @@ from buildstream import BuildElement
    57 57
    58 58 
     # Element implementation for the 'autotools' kind.
    59 59 
     class AutotoolsElement(BuildElement):
    60 
    -    pass
    60 
    +    BST_VIRTUAL_DIRECTORY = True
    61 61
    62 62
    63 63 
     # Plugin entry point

  • buildstream/sandbox/__init__.py
    ... ... @@ -20,3 +20,4 @@
    20 20 
     from .sandbox import Sandbox, SandboxFlags
    21 21 
     from ._sandboxchroot import SandboxChroot
    22 22 
     from ._sandboxbwrap import SandboxBwrap
    23 
    +from ._sandboxremote import SandboxRemote

  • buildstream/sandbox/_sandboxremote.py
    1 
    +#!/usr/bin/env python3
    2 
    +#
    3 
    +#  Copyright (C) 2018 Bloomberg LP
    4 
    +#
    5 
    +#  This program is free software; you can redistribute it and/or
    6 
    +#  modify it under the terms of the GNU Lesser General Public
    7 
    +#  License as published by the Free Software Foundation; either
    8 
    +#  version 2 of the License, or (at your option) any later version.
    9 
    +#
    10 
    +#  This library is distributed in the hope that it will be useful,
    11 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    12 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    13 
    +#  Lesser General Public License for more details.
    14 
    +#
    15 
    +#  You should have received a copy of the GNU Lesser General Public
    16 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    17 
    +#
    18 
    +#  Authors:
    19 
    +#        Jim MacArthur <jim macarthur codethink co uk>
    20 
    +
    21 
    +import os
    22 
    +import re
    23 
    +from urllib.parse import urlparse
    24 
    +
    25 
    +import grpc
    26 
    +
    27 
    +from . import Sandbox
    28 
    +from ..storage._filebaseddirectory import FileBasedDirectory
    29 
    +from ..storage._casbaseddirectory import CasBasedDirectory
    30 
    +from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    31 
    +from .._artifactcache.cascache import CASCache
    32 
    +
    33 
    +
    34 
    +class SandboxError(Exception):
    35 
    +    pass
    36 
    +
    37 
    +
    38 
    +# SandboxRemote()
    39 
    +#
    40 
    +# This isn't really a sandbox, it's a stub which sends all the source
    41 
    +# to a remote server and retrieves the results from it.
    42 
    +#
    43 
    +class SandboxRemote(Sandbox):
    44 
    +
    45 
    +    def __init__(self, *args, **kwargs):
    46 
    +        super().__init__(*args, **kwargs)
    47 
    +        self.cascache = None
    48 
    +
    49 
    +        url = urlparse(kwargs['server_url'])
    50 
    +        if not url.scheme or not url.hostname or not url.port:
    51 
    +            raise SandboxError("Configured remote URL '{}' does not match the expected layout. "
    52 
    +                               .format(self.server_url) +
    53 
    +                               "It should be of the form <protocol>://<domain name>:<port>.")
    54 
    +        elif url.scheme != 'http':
    55 
    +            raise SandboxError("Configured remote '{}' uses an unsupported protocol. "
    56 
    +                               "Only plain HTTP is currenlty supported (no HTTPS).")
    57 
    +
    58 
    +        self.server_url = '{}:{}'.format(url.hostname, url.port)
    59 
    +
    60 
    +    def _get_cascache(self):
    61 
    +        if self.cascache is None:
    62 
    +            self.cascache = CASCache(self._get_context())
    63 
    +            self.cascache.setup_remotes(use_config=True)
    64 
    +        return self.cascache
    65 
    +
    66 
    +    def run_remote_command(self, cascache, command, input_root_digest, working_directory, environment):
    67 
    +        # Sends an execution request to the remote execution server.
    68 
    +        #
    69 
    +        # This function blocks until it gets a response from the server.
    70 
    +        #
    71 
    +        environment_variables = [remote_execution_pb2.Command.
    72 
    +                                 EnvironmentVariable(name=k, value=v)
    73 
    +                                 for (k, v) in environment.items()]
    74 
    +
    75 
    +        # Create and send the Command object.
    76 
    +        remote_command = remote_execution_pb2.Command(arguments=command,
    77 
    +                                                      working_directory=working_directory,
    78 
    +                                                      environment_variables=environment_variables,
    79 
    +                                                      output_files=[],
    80 
    +                                                      output_directories=[self._output_directory],
    81 
    +                                                      platform=None)
    82 
    +        command_digest = cascache.add_object(buffer=remote_command.SerializeToString())
    83 
    +        command_ref = 'worker-command/{}'.format(command_digest.hash)
    84 
    +        cascache.set_ref(command_ref, command_digest)
    85 
    +
    86 
    +        command_push_successful = cascache.push_refs([command_ref], self._get_project(), may_have_dependencies=False)
    87 
    +        if not command_push_successful and not cascache.verify_key_pushed(command_ref, self._get_project()):
    88 
    +            # Command push failed
    89 
    +            return None
    90 
    +
    91 
    +        # Create and send the action.
    92 
    +
    93 
    +        action = remote_execution_pb2.Action(command_digest=command_digest,
    94 
    +                                             input_root_digest=input_root_digest,
    95 
    +                                             timeout=None,
    96 
    +                                             do_not_cache=True)
    97 
    +
    98 
    +        action_digest = cascache.add_object(buffer=action.SerializeToString())
    99 
    +        action_ref = 'worker-action/{}'.format(command_digest.hash)
    100 
    +        cascache.set_ref(action_ref, action_digest)
    101 
    +        action_push_successful = cascache.push_refs([action_ref], self._get_project(), may_have_dependencies=False)
    102 
    +
    103 
    +        if not action_push_successful and not cascache.verify_key_pushed(action_ref, self._get_project()):
    104 
    +            # Action push failed
    105 
    +            return None
    106 
    +
    107 
    +        # Next, try to create a communication channel to the BuildGrid server.
    108 
    +
    109 
    +        channel = grpc.insecure_channel(self.server_url)
    110 
    +        stub = remote_execution_pb2_grpc.ExecutionStub(channel)
    111 
    +        request = remote_execution_pb2.ExecuteRequest(action_digest=action_digest,
    112 
    +                                                      skip_cache_lookup=True)
    113 
    +        try:
    114 
    +            operation_iterator = stub.Execute(request)
    115 
    +        except grpc.RpcError:
    116 
    +            return None
    117 
    +
    118 
    +        operation = None
    119 
    +        with self._get_context().timed_activity("Waiting for the remote build to complete"):
    120 
    +            # It is advantageous to check operation_iterator.code() is grpc.StatusCode.OK here,
    121 
    +            # which will check the server is actually contactable. However, calling it when the
    122 
    +            # server is available seems to cause .code() to hang forever.
    123 
    +            for operation in operation_iterator:
    124 
    +                if operation.done:
    125 
    +                    break
    126 
    +
    127 
    +        return operation
    128 
    +
    129 
    +    def process_job_output(self, output_directories, output_files):
    130 
    +        # Reads the remote execution server response to an execution request.
    131 
    +        #
    132 
    +        # output_directories is an array of OutputDirectory objects.
    133 
    +        # output_files is an array of OutputFile objects.
    134 
    +        #
    135 
    +        # We only specify one output_directory, so it's an error
    136 
    +        # for there to be any output files or more than one directory at the moment.
    137 
    +        #
    138 
    +        if output_files:
    139 
    +            raise SandboxError("Output files were returned when we didn't request any.")
    140 
    +        elif not output_directories:
    141 
    +            error_text = "No output directory was returned from the build server."
    142 
    +            raise SandboxError(error_text)
    143 
    +        elif len(output_directories) > 1:
    144 
    +            error_text = "More than one output directory was returned from the build server: {}."
    145 
    +            raise SandboxError(error_text.format(output_directories))
    146 
    +
    147 
    +        digest = output_directories[0].tree_digest
    148 
    +        if digest is None or digest.hash is None or digest.hash == "":
    149 
    +            raise SandboxError("Output directory structure had no digest attached.")
    150 
    +
    151 
    +        # Now do a pull to ensure we have the necessary parts.
    152 
    +        cascache = self._get_cascache()
    153 
    +        cascache.pull_key(digest.hash, digest.size_bytes, self._get_project())
    154 
    +        path_components = os.path.split(self._output_directory)
    155 
    +
    156 
    +        # Now what we have is a digest for the output. Once we return, the calling process will
    157 
    +        # attempt to descend into our directory and find that directory, so we need to overwrite
    158 
    +        # that.
    159 
    +
    160 
    +        if not path_components:
    161 
    +            # The artifact wants the whole directory; we could just return the returned hash in its
    162 
    +            # place, but we don't have a means to do that yet.
    163 
    +            raise SandboxError("Unimplemented: Output directory is empty or equal to the sandbox root.")
    164 
    +
    165 
    +        # At the moment, we will get the whole directory back in the first directory argument and we need
    166 
    +        # to replace the sandbox's virtual directory with that. Creating a new virtual directory object
    167 
    +        # from another hash will be interesting, though...
    168 
    +
    169 
    +        new_dir = CasBasedDirectory(self._get_context(), ref=digest)
    170 
    +        self._set_virtual_directory(new_dir)
    171 
    +
    172 
    +    def run(self, command, flags, *, cwd=None, env=None):
    173 
    +        # Upload sources
    174 
    +        upload_vdir = self.get_virtual_directory()
    175 
    +
    176 
    +        if isinstance(upload_vdir, FileBasedDirectory):
    177 
    +            # Make a new temporary directory to put source in
    178 
    +            upload_vdir = CasBasedDirectory(self._get_context(), ref=None)
    179 
    +            upload_vdir.import_files(self.get_virtual_directory()._get_underlying_directory())
    180 
    +
    181 
    +        # Now, push that key (without necessarily needing a ref) to the remote.
    182 
    +        cascache = self._get_cascache()
    183 
    +
    184 
    +        ref = 'worker-source/{}'.format(upload_vdir.ref.hash)
    185 
    +        upload_vdir._save(ref)
    186 
    +        source_push_successful = cascache.push_refs([ref], self._get_project())
    187 
    +
    188 
    +        # Set up environment and working directory
    189 
    +        if cwd is None:
    190 
    +            cwd = self._get_work_directory()
    191 
    +
    192 
    +        if cwd is None:
    193 
    +            cwd = '/'
    194 
    +
    195 
    +        if env is None:
    196 
    +            env = self._get_environment()
    197 
    +
    198 
    +        # We want command args as a list of strings
    199 
    +        if isinstance(command, str):
    200 
    +            command = [command]
    201 
    +
    202 
    +        # Now transmit the command to execute
    203 
    +        if source_push_successful or cascache.verify_key_pushed(ref, self._get_project()):
    204 
    +            operation = self.run_remote_command(cascache, command, upload_vdir.ref, cwd, env)
    205 
    +
    206 
    +            if operation is None:
    207 
    +                # Failure of remote execution, usually due to an error in BuildStream
    208 
    +                # NB This error could be raised in __run_remote_command
    209 
    +                raise SandboxError("No response returned from server")
    210 
    +
    211 
    +            assert(not operation.HasField('error') and operation.HasField('response'))
    212 
    +
    213 
    +            execution_response = remote_execution_pb2.ExecuteResponse()
    214 
    +            # The response is expected to be an ExecutionResponse message
    215 
    +            assert(operation.response.Is(execution_response.DESCRIPTOR))
    216 
    +
    217 
    +            operation.response.Unpack(execution_response)
    218 
    +
    219 
    +            if execution_response.status.code != 0:
    220 
    +                # A normal error during the build: the remote execution system
    221 
    +                # has worked correctly but the command failed.
    222 
    +                # execution_response.error also contains 'message' (str) and
    223 
    +                # 'details' (iterator of Any) which we ignore at the moment.
    224 
    +                return execution_response.status.code
    225 
    +
    226 
    +            action_result = execution_response.result
    227 
    +
    228 
    +            self.process_job_output(action_result.output_directories, action_result.output_files)
    229 
    +
    230 
    +        else:
    231 
    +            raise SandboxError("Failed to verify that source has been pushed to the remote artifact cache.")
    232 
    +
    233 
    +        return 0

  • buildstream/sandbox/sandbox.py
    ... ... @@ -99,9 +99,11 @@ class Sandbox():
    99 99 
             self.__stdout = kwargs['stdout']
    100 100 
             self.__stderr = kwargs['stderr']
    101 101
    102 
    -        # Setup the directories. Root should be available to subclasses, hence
    103 
    -        # being single-underscore. The others are private to this class.
    102 
    +        # Setup the directories. Root and output_directory should be
    103 
    +        # available to subclasses, hence being single-underscore. The
    104 
    +        # others are private to this class.
    104 105 
             self._root = os.path.join(directory, 'root')
    106 
    +        self._output_directory = None
    105 107 
             self.__directory = directory
    106 108 
             self.__scratch = os.path.join(self.__directory, 'scratch')
    107 109 
             for directory_ in [self._root, self.__scratch]:
    ... ... @@ -144,11 +146,17 @@ class Sandbox():
    144 146 
                     self._vdir = FileBasedDirectory(self._root)
    145 147 
             return self._vdir
    146 148
    149 
    +    def _set_virtual_directory(self, virtual_directory):
    150 
    +        """ Sets virtual directory. Useful after remote execution
    151 
    +        has rewritten the working directory.
    152 
    +        """
    153 
    +        self._vdir = virtual_directory
    154 
    +
    147 155 
         def set_environment(self, environment):
    148 156 
             """Sets the environment variables for the sandbox
    149 157
    150 158 
             Args:
    151 
    -           directory (dict): The environment variables to use in the sandbox
    159 
    +           environment (dict): The environment variables to use in the sandbox
    152 160 
             """
    153 161 
             self.__env = environment
    154 162
    ... ... @@ -160,6 +168,15 @@ class Sandbox():
    160 168 
             """
    161 169 
             self.__cwd = directory
    162 170
    171 
    +    def set_output_directory(self, directory):
    172 
    +        """Sets the output directory - the directory which is preserved
    173 
    +        as an artifact after assembly.
    174 
    +
    175 
    +        Args:
    176 
    +           directory (str): An absolute path within the sandbox
    177 
    +        """
    178 
    +        self._output_directory = directory
    179 
    +
    163 180 
         def mark_directory(self, directory, *, artifact=False):
    164 181 
             """Marks a sandbox directory and ensures it will exist
    165 182

  • buildstream/storage/_casbaseddirectory.py
    ... ... @@ -561,3 +561,14 @@ class CasBasedDirectory(Directory):
    561 561 
             throw an exception. """
    562 562 
             raise VirtualDirectoryError("_get_underlying_directory was called on a CAS-backed directory," +
    563 563 
                                         " which has no underlying directory.")
    564 
    +
    565 
    +    def _save(self, name):
    566 
    +        """Saves this directory into the content cache as a named ref. Used
    567 
    +        by remote execution to make references for source directories so they
    568 
    +        can be pushed to a remote artifact server.
    569 
    +
    570 
    +        """
    571 
    +        self._recalculate_recursing_up()
    572 
    +        self._recalculate_recursing_down()
    573 
    +
    574 
    +        self.cas_cache.set_ref(name, self.ref)

  • doc/source/format_project.rst
    ... ... @@ -204,6 +204,24 @@ with an artifact share.
    204 204 
     You can also specify a list of caches here; earlier entries in the list
    205 205 
     will have higher priority than later ones.
    206 206
    207 
    +Remote execution
    208 
    +~~~~~~~~~~~~~~~~
    209 
    +BuildStream supports remote execution using the Google Remote Execution API
    210 
    +(REAPI). A description of how remote execution works is beyond the scope
    211 
    +of this document, but you can specify a remote server complying with the REAPI
    212 
    +using the `remote-execution` option:
    213 
    +
    214 
    +.. code:: yaml
    215 
    +
    216 
    +  remote-execution:
    217 
    +
    218 
    +    # A url defining a remote execution server
    219 
    +    url: http://buildserver.example.com:50051
    220 
    +
    221 
    +The url should contain a hostname and port separated by ':'. Only plain HTTP is
    222 
    +currently suported (no HTTPS).
    223 
    +
    224 
    +The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
    207 225
    208 226 
     .. _project_essentials_mirrors:
    209 227

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]