GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/84-bot-errors] 12 commits: README.rst: Remove badges (moved to header)

Title: GitLab

finn pushed to branch finn/84-bot-errors at BuildGrid / buildgrid

Commits:

20 changed files:

Changes:

  • README.rst
    1 1
    2 
    -.. image:: https://gitlab.com/Buildgrid/buildgrid/badges/master/pipeline.svg
    3 
    -   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    4 
    -
    5 
    -.. image:: https://gitlab.com/BuildGrid/buildgrid/badges/master/coverage.svg?job=coverage
    6 
    -   :target: https://buildgrid.gitlab.io/buildgrid/coverage
    7 
    -
    8 
    -
    9 2 
     .. _about:
    10 3
    11 
    -About BuildGrid
    12 
    -===============
    4 
    +About
    5 
    +=====
    13 6
    14 7
    15 8 
     .. _what-is-it:

  • buildgrid/_app/bots/buildbox.py
    ... ... @@ -19,7 +19,9 @@ import tempfile
    19 19
    20 20 
     from google.protobuf import any_pb2
    21 21
    22 
    +from buildgrid.settings import HASH_LENGTH
    22 23 
     from buildgrid.client.cas import upload
    24 
    +from buildgrid._exceptions import BotError
    23 25 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    24 26 
     from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
    25 27 
     from buildgrid.utils import read_file, write_file, parse_to_pb2_from_fetch
    ... ... @@ -87,17 +89,30 @@ def work_buildbox(context, lease):
    87 89
    88 90 
                 command_line = subprocess.Popen(command_line,
    89 91 
                                                 stdin=subprocess.PIPE,
    90 
    -                                            stdout=subprocess.PIPE)
    91 
    -            # TODO: Should return the stdout and stderr to the user.
    92 
    -            command_line.communicate()
    92 
    +                                            stdout=subprocess.PIPE,
    93 
    +                                            stderr=subprocess.PIPE)
    94 
    +            stdout, stderr = command_line.communicate()
    95 
    +            action_result = remote_execution_pb2.ActionResult()
    96 
    +            # TODO: Upload to CAS or output RAW
    97 
    +            # For now, just pass raw
    98 
    +            # https://gitlab.com/BuildGrid/buildgrid/issues/90
    99 
    +            action_result.stdout_raw = stdout
    100 
    +
    101 
    +            if stderr:
    102 
    +                # TODO: Upload to CAS or output RAW
    103 
    +                # For now, just pass raw
    104 
    +                # https://gitlab.com/BuildGrid/buildgrid/issues/90
    105 
    +                logger.error("BuildBox error: [{}]".format(stderr))
    106 
    +                raise BotError(stderr, detail=stdout, reason="Captured stderr")
    93 107
    94 108 
                 output_digest = remote_execution_pb2.Digest()
    95 109 
                 output_digest.ParseFromString(read_file(output_digest_file.name))
    96 110
    97 111 
                 logger.debug("Output root digest: {}".format(output_digest))
    98 112
    99 
    -            if len(output_digest.hash) < 64:
    100 
    -                logger.warning("Buildbox command failed - no output root digest present.")
    113 
    +            if len(output_digest.hash) < HASH_LENGTH:
    114 
    +                raise BotError("Output hash length too small",
    115 
    +                               detail=stdout, reason="No output root digest present.")
    101 116
    102 117 
                 # TODO: Have BuildBox helping us creating the Tree instance here
    103 118 
                 # See https://gitlab.com/BuildStream/buildbox/issues/7 for details
    ... ... @@ -110,7 +125,6 @@ def work_buildbox(context, lease):
    110 125 
                 output_directory.tree_digest.CopyFrom(output_tree_digest)
    111 126 
                 output_directory.path = os.path.relpath(working_directory, start='/')
    112 127
    113 
    -            action_result = remote_execution_pb2.ActionResult()
    114 128 
                 action_result.output_directories.extend([output_directory])
    115 129
    116 130 
                 action_result_any = any_pb2.Any()

  • buildgrid/_app/bots/temp_directory.py
    ... ... @@ -77,11 +77,20 @@ def work_temp_directory(context, lease):
    77 77 
                                        universal_newlines=True,
    78 78 
                                        env=environment,
    79 79 
                                        stdin=subprocess.PIPE,
    80 
    -                                   stdout=subprocess.PIPE)
    81 
    -        # TODO: Should return the stdout and stderr in the ActionResult.
    82 
    -        process.communicate()
    80 
    +                                   stdout=subprocess.PIPE,
    81 
    +                                   stderr=subprocess.PIPE)
    82 
    +        stdout, stderr = process.communicate()
    83 83
    84 84 
             action_result = remote_execution_pb2.ActionResult()
    85 
    +        # TODO: Upload to CAS or output RAW
    86 
    +        # For now, just pass raw
    87 
    +        # https://gitlab.com/BuildGrid/buildgrid/issues/90
    88 
    +        action_result.stdout_raw = stdout
    89 
    +        action_result.stderr_raw = stderr
    90 
    +
    91 
    +        if stderr:
    92 
    +            logger.error("Bot error: [{}]".format(stderr))
    93 
    +            raise BotError(stderr, detail=stdout, reason="Captured stderr")
    85 94
    86 95 
             with upload(context.cas_channel, instance=instance_name) as cas:
    87 96 
                 for output_path in command.output_files:

  • buildgrid/_app/commands/cmd_execute.py
    ... ... @@ -97,20 +97,6 @@ def request_dummy(context, number, wait_for_completion):
    97 97 
                 context.logger.info(next(response))
    98 98
    99 99
    100 
    -@cli.command('wait', short_help="Streams an operation until it is complete.")
    101 
    -@click.argument('operation-name', nargs=1, type=click.STRING, required=True)
    102 
    -@pass_context
    103 
    -def wait_execution(context, operation_name):
    104 
    -    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
    105 
    -    request = remote_execution_pb2.WaitExecutionRequest(instance_name=context.instance_name,
    106 
    -                                                        name=operation_name)
    107 
    -
    108 
    -    response = stub.WaitExecution(request)
    109 
    -
    110 
    -    for stream in response:
    111 
    -        context.logger.info(stream)
    112 
    -
    113 
    -
    114 100 
     @cli.command('command', short_help="Send a command to be executed.")
    115 101 
     @click.option('--output-file', nargs=2, type=(click.STRING, click.BOOL), multiple=True,
    116 102 
                   help="Tuple of expected output file and is-executeable flag.")

  • buildgrid/_app/commands/cmd_operation.py
    ... ... @@ -27,12 +27,13 @@ import sys
    27 27 
     import click
    28 28 
     import grpc
    29 29
    30 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    30 31 
     from buildgrid._protos.google.longrunning import operations_pb2, operations_pb2_grpc
    31 32
    32 33 
     from ..cli import pass_context
    33 34
    34 35
    35 
    -@click.group(name='operation', short_help="Long running operations commands")
    36 
    +@click.group(name='operation', short_help="Long running operations commands.")
    36 37 
     @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    37 38 
                   help="Remote execution server's URL (port defaults to 50051 if no specified).")
    38 39 
     @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    ... ... @@ -93,3 +94,17 @@ def lists(context):
    93 94
    94 95 
         for op in response.operations:
    95 96 
             context.logger.info(op)
    97 
    +
    98 
    +
    99 
    +@cli.command('wait', short_help="Streams an operation until it is complete.")
    100 
    +@click.argument('operation-name', nargs=1, type=click.STRING, required=True)
    101 
    +@pass_context
    102 
    +def wait(context, operation_name):
    103 
    +    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
    104 
    +    request = remote_execution_pb2.WaitExecutionRequest(instance_name=context.instance_name,
    105 
    +                                                        name=operation_name)
    106 
    +
    107 
    +    response = stub.WaitExecution(request)
    108 
    +
    109 
    +    for stream in response:
    110 
    +        context.logger.info(stream)

  • buildgrid/_app/settings/cas.yml
    ... ... @@ -17,7 +17,7 @@ instances:
    17 17
    18 18 
         storages:
    19 19 
             - !disk-storage &main-storage
    20 
    -          path: ~/cas/
    20 
    +          path: !expand-path $HOME/cas/
    21 21
    22 22 
         services:
    23 23 
           - !cas

  • buildgrid/_app/settings/default.yml
    ... ... @@ -17,7 +17,7 @@ instances:
    17 17
    18 18 
         storages:
    19 19 
             - !disk-storage &main-storage
    20 
    -          path: ~/cas/
    20 
    +          path: !expand-path $HOME/cas/
    21 21
    22 22 
         services:
    23 23 
           - !action-cache &main-action

  • buildgrid/_app/settings/parser.py
    ... ... @@ -68,12 +68,21 @@ class Channel(YamlFactory):
    68 68 
                     sys.exit(-1)
    69 69
    70 70
    71 
    +class ExpandPath(YamlFactory):
    72 
    +
    73 
    +    yaml_tag = u'!expand-path'
    74 
    +
    75 
    +    def __new__(cls, path):
    76 
    +        path = os.path.expanduser(path)
    77 
    +        path = os.path.expandvars(path)
    78 
    +        return path
    79 
    +
    80 
    +
    71 81 
     class Disk(YamlFactory):
    72 82
    73 83 
         yaml_tag = u'!disk-storage'
    74 84
    75 85 
         def __new__(cls, path):
    76 
    -        path = os.path.expanduser(path)
    77 86 
             return DiskStorage(path)
    78 87
    79 88
    ... ... @@ -197,6 +206,7 @@ def _parse_size(size):
    197 206 
     def get_parser():
    198 207
    199 208 
         yaml.SafeLoader.add_constructor(Channel.yaml_tag, Channel.from_yaml)
    209 
    +    yaml.SafeLoader.add_constructor(ExpandPath.yaml_tag, ExpandPath.from_yaml)
    200 210 
         yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
    201 211 
         yaml.SafeLoader.add_constructor(Action.yaml_tag, Action.from_yaml)
    202 212 
         yaml.SafeLoader.add_constructor(Reference.yaml_tag, Reference.from_yaml)

  • buildgrid/_app/settings/remote-storage.yml
    ... ... @@ -19,10 +19,10 @@ instances:
    19 19 
             - !remote-storage &main-storage
    20 20 
               url: "http://localhost:50052"
    21 21 
               instance_name: main
    22 
    -          credentials:
    23 
    -            tls-client-key: null
    24 
    -            tls-client-cert: null
    25 
    -            tls-server-cert: null
    22 
    +#          credentials:
    23 
    +#            tls-client-key: null
    24 
    +#            tls-client-cert: null
    25 
    +#            tls-server-cert: null
    26 26
    27 27 
         services:
    28 28 
           - !action-cache &main-action

  • buildgrid/bot/bot_session.py
    ... ... @@ -12,6 +12,9 @@
    12 12 
     # See the License for the specific language governing permissions and
    13 13 
     # limitations under the License.
    14 14
    15 
    +# Disable broad exception catch
    16 
    +# pylint: disable=broad-except
    17 
    +
    15 18
    16 19 
     """
    17 20 
     Bot Session
    ... ... @@ -23,10 +26,14 @@ import asyncio
    23 26 
     import logging
    24 27 
     import platform
    25 28 
     import uuid
    26 
    -
    27 29 
     from enum import Enum
    28 30
    31 
    +import grpc
    32 
    +from google.protobuf import any_pb2
    33 
    +
    29 34 
     from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2, worker_pb2
    35 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    36 
    +from buildgrid._exceptions import BotError
    30 37
    31 38
    32 39 
     class BotStatus(Enum):
    ... ... @@ -142,13 +149,35 @@ class BotSession:
    142 149
    143 150 
         async def create_work(self, lease):
    144 151 
             self.logger.debug("Work created: [{}]".format(lease.id))
    145 
    -
    152 
    +        input_lease = lease
    146 153 
             loop = asyncio.get_event_loop()
    147 
    -        lease = await loop.run_in_executor(None, self._work, self._context, lease)
    154 
    +
    155 
    +        try:
    156 
    +            lease = await loop.run_in_executor(None, self._work, self._context, lease)
    157 
    +
    158 
    +        except BotError as e:
    159 
    +            self.logger.error("Bot error thrown: [{}]".format(e))
    160 
    +            lease = self._lease_error(input_lease, e)
    161 
    +
    162 
    +        except grpc.RpcError as e:
    163 
    +            self.logger.error("Connection error thrown: [{}]".format(e))
    164 
    +            lease = self._lease_error(input_lease, e)
    165 
    +
    166 
    +        except Exception as e:
    167 
    +            self.logger.error("Connection error thrown: [{}]".format(e))
    168 
    +            lease = self._lease_error(input_lease, e)
    148 169
    149 170 
             self.logger.debug("Work complete: [{}]".format(lease.id))
    150 171 
             self.lease_completed(lease)
    151 172
    173 
    +    def _lease_error(self, lease, error):
    174 
    +        action_result = remote_execution_pb2.ActionResult()
    175 
    +        action_result.stderr_raw = str(error)
    176 
    +        action_result_any = any_pb2.Any()
    177 
    +        action_result_any.Pack(action_result)
    178 
    +        lease.result.CopyFrom(action_result_any)
    179 
    +        return lease
    180 
    +
    152 181
    153 182 
     class Worker:
    154 183 
         def __init__(self, properties=None, configs=None):

  • buildgrid/server/job.py
    ... ... @@ -21,6 +21,7 @@ from enum import Enum
    21 21
    22 22 
     from google.protobuf import any_pb2
    23 23
    24 
    +from buildgrid._protos.google.rpc import code_pb2, status_pb2
    24 25 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    25 26 
     from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2
    26 27 
     from buildgrid._protos.google.longrunning import operations_pb2
    ... ... @@ -121,10 +122,14 @@ class Job:
    121 122 
             self._operation.metadata.CopyFrom(self._pack_any(self.get_operation_meta()))
    122 123 
             if self.result is not None:
    123 124 
                 self._operation.done = True
    124 
    -            action_result = remote_execution_pb2.ActionResult()
    125 
    -            self.result.Unpack(action_result)
    126 
    -            response = remote_execution_pb2.ExecuteResponse(result=action_result,
    127 
    -                                                            cached_result=self.result_cached)
    125 
    +            status = status_pb2.Status()
    126 
    +            status.code = code_pb2.OK
    127 
    +            if self.result.stderr_raw or self.result.stderr_digest:
    128 
    +                status.code = code_pb2.INTERNAL
    129 
    +
    130 
    +            response = remote_execution_pb2.ExecuteResponse(result=self.result,
    131 
    +                                                            cached_result=self.result_cached,
    132 
    +                                                            status=status)
    128 133 
                 self._operation.response.CopyFrom(self._pack_any(response))
    129 134
    130 135 
             return self._operation

  • buildgrid/server/scheduler.py
    ... ... @@ -27,6 +27,7 @@ from google.protobuf import any_pb2
    27 27
    28 28
    29 29 
     from buildgrid.server._exceptions import NotFoundError
    30 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    30 31 
     from buildgrid._protos.google.longrunning import operations_pb2
    31 32
    32 33 
     from .job import ExecuteStage, LeaseState
    ... ... @@ -84,10 +85,13 @@ class Scheduler:
    84 85
    85 86 
         def job_complete(self, name, result):
    86 87 
             job = self.jobs[name]
    87 
    -        job.result = result
    88 
    -        job.update_execute_stage(ExecuteStage.COMPLETED)
    88 
    +        action_result = remote_execution_pb2.ActionResult()
    89 
    +        result.Unpack(action_result)
    90 
    +        job.result = action_result
    89 91 
             if not job.do_not_cache and self._action_cache is not None:
    90 
    -            self._action_cache.update_action_result(job.action_digest, result)
    92 
    +            if not (action_result.stderr_raw or action_result.stderr_digest):
    93 
    +                self._action_cache.update_action_result(job.action_digest, result)
    94 
    +        job.update_execute_stage(ExecuteStage.COMPLETED)
    91 95
    92 96 
         def get_operations(self):
    93 97 
             response = operations_pb2.ListOperationsResponse()

  • buildgrid/settings.py
    ... ... @@ -3,3 +3,4 @@ import hashlib
    3 3
    4 4 
     # The hash function that CAS uses
    5 5 
     HASH = hashlib.sha256
    6 
    +HASH_LENGTH = 64

  • docs/source/data/bazel-example-server.conf
    1 1 
     server:
    2 
    -  port: 50051
    3 
    -  insecure-mode: true
    2 
    +  - !channel
    3 
    +    port: 50051
    4 
    +    insecure_mode: true
    4 5
    5 6 
     instances:
    6 7 
       - name: main

  • docs/source/data/buildstream-example-server.conf
    1 
    +server:
    2 
    +  - !channel
    3 
    +    port: 50051
    4 
    +    insecure_mode: true
    5 
    +
    6 
    +instances:
    7 
    +  - name: ""
    8 
    +
    9 
    +    storages:
    10 
    +      - !lru-storage &main-storage
    11 
    +        size: 512MB
    12 
    +
    13 
    +    services:
    14 
    +      - !action-cache &main-action
    15 
    +        storage: *main-storage
    16 
    +        max_cached_refs: 256
    17 
    +        allow_updates: true
    18 
    +      - !execution
    19 
    +        storage: *main-storage
    20 
    +        action_cache: *main-action
    21 
    +      - !cas
    22 
    +        storage: *main-storage
    23 
    +      - !reference-cache
    24 
    +        storage: *main-storage
    25 
    +        max_cached_refs: 128
    26 
    +      - !bytestream
    27 
    +        storage: *main-storage
    \ No newline at end of file

  • docs/source/reference_cli.rst
    1 1
    2 
    -.. _cli_reference:
    2 
    +.. _cli-reference:
    3 3
    4 4 
     CLI reference
    5 5 
     =============
    ... ... @@ -8,112 +8,118 @@ BuildGrid's Command Line Interface (CLI) reference documentation.
    8 8
    9 9 
     ----
    10 10
    11 
    -.. _invoking_bgd:
    11 
    +.. _invoking-bgd:
    12 12
    13 13 
     .. click:: buildgrid._app:cli
    14 14 
        :prog: bgd
    15 15
    16 16 
     ----
    17 17
    18 
    -.. _invoking_bgd_bot:
    18 
    +.. _invoking-bgd-bot:
    19 19
    20 20 
     .. click:: buildgrid._app.commands.cmd_bot:cli
    21 21 
        :prog: bgd bot
    22 22
    23 23 
     ----
    24 24
    25 
    -.. _invoking_bgd_bot_buildbox:
    25 
    +.. _invoking-bgd-bot-buildbox:
    26 26
    27 27 
     .. click:: buildgrid._app.commands.cmd_bot:run_buildbox
    28 28 
        :prog: bgd bot buildbot
    29 29
    30 30 
     ----
    31 31
    32 
    -.. _invoking_bgd_bot_dummy:
    32 
    +.. _invoking-bgd-bot-dummy:
    33 33
    34 34 
     .. click:: buildgrid._app.commands.cmd_bot:run_dummy
    35 35 
        :prog: bgd bot dummy
    36 36
    37 37 
     ----
    38 38
    39 
    -.. _invoking_bgd_bot_temp_directory:
    39 
    +.. _invoking-bgd-bot-temp-directory:
    40 40
    41 41 
     .. click:: buildgrid._app.commands.cmd_bot:run_temp_directory
    42 42 
        :prog: bgd bot temp-directory
    43 43
    44 44 
     ----
    45 45
    46 
    -.. _invoking_bgd_cas:
    46 
    +.. _invoking-bgd-cas:
    47 47
    48 48 
     .. click:: buildgrid._app.commands.cmd_cas:cli
    49 49 
        :prog: bgd cas
    50 50
    51 51 
     ----
    52 52
    53 
    -.. _invoking_bgd_cas_upload_dir:
    53 
    +.. _invoking-bgd-cas-upload-dir:
    54 54
    55 55 
     .. click:: buildgrid._app.commands.cmd_cas:upload_dir
    56 56 
        :prog: bgd cas upload-dir
    57 57
    58 58 
     ----
    59 59
    60 
    -.. _invoking_bgd_cas_upload_files:
    60 
    +.. _invoking-bgd-cas-upload-files:
    61 61
    62 62 
     .. click:: buildgrid._app.commands.cmd_cas:upload_files
    63 63 
        :prog: bgd cas upload-files
    64 64
    65 65 
     ----
    66 66
    67 
    -.. _invoking_bgd_execute:
    67 
    +.. _invoking-bgd-execute:
    68 68
    69 69 
     .. click:: buildgrid._app.commands.cmd_execute:cli
    70 70 
        :prog: bgd execute
    71 71
    72 72 
     ----
    73 73
    74 
    -.. _invoking_bgd_execute_command:
    74 
    +.. _invoking-bgd-execute-command:
    75 75
    76 76 
     .. click:: buildgrid._app.commands.cmd_execute:command
    77 77 
        :prog: bgd execute command
    78 78
    79 79 
     ----
    80 80
    81 
    -.. _invoking_bgd_execute_list:
    81 
    +.. _invoking-bgd-execute-request-dummy:
    82 82
    83 
    -.. click:: buildgrid._app.commands.cmd_execute:list_operations
    84 
    -   :prog: bgd execute list
    83 
    +.. click:: buildgrid._app.commands.cmd_execute:request_dummy
    84 
    +   :prog: bgd execute request-dummy
    85 85
    86 86 
     ----
    87 87
    88 
    -.. _invoking_bgd_execute_ request_dummy:
    88 
    +.. _invoking-bgd-operation:
    89 89
    90 
    -.. click:: buildgrid._app.commands.cmd_execute:request_dummy
    91 
    -   :prog: bgd execute request-dummy
    90 
    +.. click:: buildgrid._app.commands.cmd_operation:cli
    91 
    +   :prog: bgd operation
    92 92
    93 93 
     ----
    94 94
    95 
    -.. _invoking_bgd_execute_status:
    95 
    +.. _invoking-bgd-operation-list:
    96 96
    97 
    -.. click:: buildgrid._app.commands.cmd_execute:operation_status
    98 
    -   :prog: bgd execute status
    97 
    +.. click:: buildgrid._app.commands.cmd_operation:lists
    98 
    +   :prog: bgd operation list
    99 99
    100 100 
     ----
    101 101
    102 
    -.. _invoking_bgd_execute_wait:
    102 
    +.. _invoking-bgd-operation-status:
    103 103
    104 
    -.. click:: buildgrid._app.commands.cmd_execute:wait_execution
    105 
    -   :prog: bgd execute wait
    104 
    +.. click:: buildgrid._app.commands.cmd_operation:status
    105 
    +   :prog: bgd operation status
    106 106
    107 107 
     ----
    108 108
    109 
    -.. _invoking_bgd_server:
    109 
    +.. _invoking-bgd-operation-wait:
    110 
    +
    111 
    +.. click:: buildgrid._app.commands.cmd_operation:wait
    112 
    +   :prog: bgd operation wait
    113 
    +
    114 
    +----
    115 
    +.. _invoking-bgd-server:
    110 116
    111 117 
     .. click:: buildgrid._app.commands.cmd_server:cli
    112 118 
        :prog: bgd server
    113 119
    114 120 
     ----
    115 121
    116 
    -.. _invoking_bgd_server_start:
    122 
    +.. _invoking-bgd-server-start:
    117 123
    118 124 
     .. click:: buildgrid._app.commands.cmd_server:start
    119 125 
        :prog: bgd server start

  • docs/source/using.rst
    ... ... @@ -7,7 +7,8 @@ Using
    7 7 
     This section covers how to run an use the BuildGrid build service.
    8 8
    9 9 
     .. toctree::
    10 
    -   :maxdepth: 2
    10 
    +   :maxdepth: 3
    11 11
    12 12 
        using_internal.rst
    13 13 
        using_bazel.rst
    14 
    +   using_buildstream.rst

  • docs/source/using_bazel.rst
    1 1
    2 
    -.. _bazel-builds:
    2 
    +.. _bazel-client:
    3 3
    4 
    -Bazel builds
    4 
    +Bazel client
    5 5 
     ============
    6 6
    7 7 
     `Bazel`_ is a *“fast, scalable, multi-language and extensible build system”*
    ... ... @@ -18,7 +18,7 @@ Configuration
    18 18 
     -------------
    19 19
    20 20 
     Bazel accepts many options that can either be specified as command line
    21 
    -arguments when involking the ``bazel`` tool or stored in a `.bazelrc`_
    21 
    +arguments when invoking the ``bazel`` tool or stored in a `.bazelrc`_
    22 22 
     configuration file. In order to activate remote build execution, the
    23 23 
     ``bazel build`` subcommand needs to be given specific `build options`_,
    24 24 
     including:
    ... ... @@ -65,7 +65,7 @@ Next, change the current directory to the Bazel workspace root:
    65 65
    66 66 
        cd bazel-examples/cpp-tutorial/stage3
    67 67
    68 
    -.. note::
    68 
    +.. hint::
    69 69
    70 70 
        All the commands in the instructions below are expected to be executed from
    71 71 
        that root directory (the stage3 example workspace's root directory).
    ... ... @@ -80,7 +80,7 @@ below, paste it in a ``server.conf`` file in the root directory:
    80 80 
     This defines a single ``main`` server instance implementing a
    81 81 
     ``ContentAddressableStorage`` (CAS) + ``ByteStream`` service together with an
    82 82 
     ``Execution`` + ``ActionCache`` service, both using the same in-memory storage.
    83 
    -You can then start the BuildGrid server deamon using that configuration by
    83 
    +You can then start the BuildGrid server daemon using that configuration by
    84 84 
     running:
    85 85
    86 86 
     .. code-block:: sh
    ... ... @@ -97,8 +97,10 @@ has ``gcc`` installed, run:
    97 97 
        bgd bot --remote=http://localhost:50051 --parent=main temp-directory
    98 98
    99 99 
     The ``--remote`` option is used to specify the server location (running on the
    100 
    -same machine here, and listenning to port 50051). The ``--parent`` option is
    101 
    -used to specifiy the server instance you except the bot to be attached to.
    100 
    +same machine here, and listening to port 50051). The ``--parent`` option is used
    101 
    +to specify the server instance you except the bot to be attached to. Refer to
    102 
    +the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
    103 
    +line interface details.
    102 104
    103 105 
     The BuildGrid server is now ready to accept jobs and execute them. Bazel needs
    104 106 
     some :ref:`configuration <bazel-configuration>` in order to run remote builds.
    ... ... @@ -109,7 +111,7 @@ file in the root directory:
    109 111
    110 112 
        build --spawn_strategy=remote --genrule_strategy=remote --remote_executor=localhost:50051 --remote_instance_name=main
    111 113
    112 
    -This ativates Bazel's remote execution mode and points to the ``main`` remote
    114 
    +This activates Bazel's remote execution mode and points to the ``main`` remote
    113 115 
     execution server instance at ``localhost:50051``.
    114 116
    115 117 
     You can finally have Bazel to build the example workspace by running:
    ... ... @@ -118,7 +120,7 @@ You can finally have Bazel to build the example workspace by running:
    118 120
    119 121 
        bazel build //main:hello-world
    120 122
    121 
    -You can verify that the example has been successfully build by runnig the
    123 
    +You can verify that the example has been successfully build by running the
    122 124 
     generated executable. Simply invoke:
    123 125
    124 126 
     .. code-block:: sh

  • docs/source/using_buildstream.rst
    1 
    +
    2 
    +.. _buildstream-client:
    3 
    +
    4 
    +BuildStream client
    5 
    +==================
    6 
    +
    7 
    +`BuildStream`_  is a free software tool for building and integrating software
    8 
    +stacks. It supports remote build execution using the remote execution API
    9 
    +(REAPI) v2.
    10 
    +
    11 
    +.. note::
    12 
    +
    13 
    +   There is no stable release of BuildStream with support for remote execution
    14 
    +   yet. You'll have to `install it from sources`_ in order to build with
    15 
    +   remote execution.
    16 
    +
    17 
    +.. _BuildStream: https://buildstream.build
    18 
    +.. _install it from sources: https://buildstream.build/source_install.html
    19 
    +
    20 
    +
    21 
    +.. _buildstream-configuration:
    22 
    +
    23 
    +Configuration
    24 
    +-------------
    25 
    +
    26 
    +BuildStream uses `YAML`_ for build definition and configuration. It has two
    27 
    +levels of configuration: user and project level. `User-level configuration`_ is
    28 
    +stored in your ``buildstream.conf`` file while `project-level configuration`_
    29 
    +is defined in each project's ``project.conf`` file.
    30 
    +
    31 
    +.. note::
    32 
    +
    33 
    +   At the moment, remote execution can only be configured at project-level.
    34 
    +
    35 
    +.. _YAML: http://yaml.org
    36 
    +.. _User-level configuration: https://buildstream.gitlab.io/buildstream/using_config.html
    37 
    +.. _project-level configuration: https://buildstream.gitlab.io/buildstream/format_project.html
    38 
    +
    39 
    +
    40 
    +Project configuration
    41 
    +~~~~~~~~~~~~~~~~~~~~~
    42 
    +
    43 
    +In order to activate remote build execution at project-level, the project's
    44 
    +``project.conf`` file must declare two specific configuration nodes:
    45 
    +
    46 
    +- ``artifacts`` for `remote CAS endpoint details`_.
    47 
    +- ``remote-execution`` for `remote execution endpoint details`_.
    48 
    +
    49 
    +.. important::
    50 
    +
    51 
    +   BuildStream does not support multi-instance remote execution servers and will
    52 
    +   always submit remote execution request omitting the instance name parameter.
    53 
    +   Thus, you must declare an unnamed `""` instance  in your server configuration
    54 
    +   to workaround this.
    55 
    +
    56 
    +.. important::
    57 
    +
    58 
    +   If you are using BuildGrid's artifact server, the server instance **must**
    59 
    +   accept pushes from your client for remote execution to be possible.
    60 
    +
    61 
    +
    62 
    +.. _remote CAS endpoint details: https://buildstream.gitlab.io/buildstream/install_artifacts.html#user-configuration
    63 
    +.. _remote execution endpoint details: https://buildstream.gitlab.io/buildstream/format_project.html#remote-execution
    64 
    +
    65 
    +
    66 
    +BuildBox tool
    67 
    +~~~~~~~~~~~~~
    68 
    +
    69 
    +BuildStream performs builds in a `sandbox`_ on top of a project-defined
    70 
    +environment, not relying on any host-tools. BuildGrid supports this kind of
    71 
    +build using the ``buildbox`` bot, a specific type of bot relying on `BuildBox`_
    72 
    +for build execution.
    73 
    +
    74 
    +BuildBox can execute build commands in a sandbox on top of an environment
    75 
    +constructed from provided sources using `FUSE`_. It also uses `bubblewrap`_
    76 
    +for sandboxing without requiring root privileges.
    77 
    +
    78 
    +BuildBox being a rather young project, it isn't packaged yet and you'll have to
    79 
    +build it from source. You may want follow the `manual instructions`_ or you can
    80 
    +build it with BuildStream using the `dedicated integration project`_ (recommanded).
    81 
    +
    82 
    +.. important::
    83 
    +
    84 
    +   Whatever the method you use to install BuildBox, you also have to install
    85 
    +   bubblewrap along, minimum required version being `0.1.8`_.
    86 
    +
    87 
    +.. _sandbox: https://buildstream.gitlab.io/buildstream/additional_sandboxing.html
    88 
    +.. _BuildBox: https://gitlab.com/BuildStream/buildbox
    89 
    +.. _FUSE: https://en.wikipedia.org/wiki/Filesystem_in_Userspace
    90 
    +.. _bubblewrap: https://github.com/projectatomic/bubblewrap
    91 
    +.. _manual instructions: https://gitlab.com/BuildStream/buildbox/blob/master/INSTALL.rst
    92 
    +.. _dedicated integration project: https://gitlab.com/BuildStream/buildbox-integration
    93 
    +.. _0.1.8: https://github.com/projectatomic/bubblewrap/releases/tag/v0.1.8
    94 
    +
    95 
    +
    96 
    +.. _buildstream-example:
    97 
    +
    98 
    +Example build
    99 
    +-------------
    100 
    +
    101 
    +The BuildStream repository contains `example projects`_ used for testing purpose
    102 
    +in the project's `usage documentation section`_. We'll focus here on
    103 
    +instructions on how to build the `autotools example`_ running BuildStream and
    104 
    +BuildGrid on your local machine, compiling the `GNU Automake`_ hello example
    105 
    +program in a sandbox on top of a minimal `Alpine Linux`_ environment.
    106 
    +
    107 
    +First, you need to checkout the buildstream repository sources:
    108 
    +
    109 
    +.. code-block:: sh
    110 
    +
    111 
    +   git clone https://gitlab.com/BuildStream/buildstream.git
    112 
    +
    113 
    +Next, change the current directory to the BuildStream project root:
    114 
    +
    115 
    +.. code-block:: sh
    116 
    +
    117 
    +   cd buildstream/doc/examples/autotools
    118 
    +
    119 
    +.. hint::
    120 
    +
    121 
    +   All the commands in the instructions below are expected to be executed from
    122 
    +   that root directory (the autotools example project's root directory).
    123 
    +
    124 
    +Before running BuildStream and building the example project, you'll have to setup
    125 
    +and run a BuildGrid server and bot. A minimal server's configuration is given
    126 
    +below, paste it in a ``server.conf`` file in the root directory:
    127 
    +
    128 
    +.. literalinclude:: ./data/buildstream-example-server.conf
    129 
    +   :language: yaml
    130 
    +
    131 
    +This defines a single unnamed server instance implementing a
    132 
    +``ContentAddressableStorage`` (CAS) + ``Reference`` + ``ByteStream`` service
    133 
    +together with an ``Execution`` + ``ActionCache`` service, both using the
    134 
    +same in-memory storage. You can then start the BuildGrid server daemon using
    135 
    +that configuration by running:
    136 
    +
    137 
    +.. code-block:: sh
    138 
    +
    139 
    +   bgd server start server.conf
    140 
    +
    141 
    +In order to perform the actual build work, you need to attach a ``buildbox``
    142 
    +worker bot to that server for that unnamed instance. Once you've make sure
    143 
    +that the ``buildbox`` tool is functional on your machine (refer to
    144 
    +:ref:`configuration <buildstream-configuration>`), run:
    145 
    +
    146 
    +.. code-block:: sh
    147 
    +
    148 
    +   bgd bot --remote=http://localhost:50051 --parent= buildbox --fuse-dir=fuse --local-cas=cache
    149 
    +
    150 
    +The ``--remote`` option is used to specify the server location (running on the
    151 
    +same machine here, and listening to port 50051). The ``--parent`` option is
    152 
    +used to specify the server instance you except the bot to be attached to (empty
    153 
    +here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
    154 
    +and respectively specify the sandbox mount point and local CAS cache locations.
    155 
    +Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
    156 
    +for command line interface details.
    157 
    +
    158 
    +The BuildGrid server is now ready to accept jobs and execute them. The example
    159 
    +project needs some :ref:`configuration <buildstream-configuration>` tweaks in
    160 
    +order to be build remotely. Below is the configuration fragment you should
    161 
    +append at the end of the ``project.conf`` file from the root directory:
    162 
    +
    163 
    +.. code-block:: yaml
    164 
    +
    165 
    +   artifacts:
    166 
    +     url: http://localhost:50051
    167 
    +     push: true
    168 
    +
    169 
    +   remote-execution:
    170 
    +     url: http://localhost:50051
    171 
    +
    172 
    +This activates BuildGrid's remote execution mode and points to the unnamed
    173 
    +remote execution server instance at ``localhost:50051``.
    174 
    +
    175 
    +You can finally have BuildStream to build the example project by running:
    176 
    +
    177 
    +.. code-block:: sh
    178 
    +
    179 
    +   bst build hello.bst
    180 
    +
    181 
    +You can verify that the example has been successfully build by running the
    182 
    +generated executable. Simply invoke:
    183 
    +
    184 
    +.. code-block:: sh
    185 
    +
    186 
    +   bst shell hello.bst -- hello
    187 
    +
    188 
    +.. _example projects: https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples
    189 
    +.. _usage documentation section: http://buildstream.gitlab.io/buildstream/main_using.html
    190 
    +.. _autotools example: https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/autotools
    191 
    +.. _GNU Automake: https://www.gnu.org/software/automake
    192 
    +.. _Alpine Linux: https://www.alpinelinux.org

  • docs/source/using_internal.rst
    1 1
    2 
    -.. _internal-builds:
    2 
    +.. _internal-client:
    3 3
    4 
    -Internal builds
    4 
    +Internal client
    5 5 
     ===============
    6 6
    7 
    +BuildGrid contains a minimal remote execution client that can be used through
    8 
    +the ``bgd`` command line interface.
    9 
    +
    7 10
    8 11 
     .. _dummy-test:
    9 12

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]