GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/51-clients-usage] 3 commits: _app/commands: Move 'wait' from 'execute' to 'operation'

Title: GitLab

Martin Blanchard pushed to branch mablanch/51-clients-usage at BuildGrid / buildgrid

Commits:

8 changed files:

Changes:

  • buildgrid/_app/commands/cmd_execute.py
    ... ... @@ -97,20 +97,6 @@ def request_dummy(context, number, wait_for_completion):
    97 97 
                 context.logger.info(next(response))
    98 98
    99 99
    100 
    -@cli.command('wait', short_help="Streams an operation until it is complete.")
    101 
    -@click.argument('operation-name', nargs=1, type=click.STRING, required=True)
    102 
    -@pass_context
    103 
    -def wait_execution(context, operation_name):
    104 
    -    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
    105 
    -    request = remote_execution_pb2.WaitExecutionRequest(instance_name=context.instance_name,
    106 
    -                                                        name=operation_name)
    107 
    -
    108 
    -    response = stub.WaitExecution(request)
    109 
    -
    110 
    -    for stream in response:
    111 
    -        context.logger.info(stream)
    112 
    -
    113 
    -
    114 100 
     @cli.command('command', short_help="Send a command to be executed.")
    115 101 
     @click.option('--output-file', nargs=2, type=(click.STRING, click.BOOL), multiple=True,
    116 102 
                   help="Tuple of expected output file and is-executeable flag.")

  • buildgrid/_app/commands/cmd_operation.py
    ... ... @@ -32,7 +32,7 @@ from buildgrid._protos.google.longrunning import operations_pb2, operations_pb2_
    32 32 
     from ..cli import pass_context
    33 33
    34 34
    35 
    -@click.group(name='operation', short_help="Long running operations commands")
    35 
    +@click.group(name='operation', short_help="Long running operations commands.")
    36 36 
     @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    37 37 
                   help="Remote execution server's URL (port defaults to 50051 if no specified).")
    38 38 
     @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    ... ... @@ -93,3 +93,17 @@ def lists(context):
    93 93
    94 94 
         for op in response.operations:
    95 95 
             context.logger.info(op)
    96 
    +
    97 
    +
    98 
    +@cli.command('wait', short_help="Streams an operation until it is complete.")
    99 
    +@click.argument('operation-name', nargs=1, type=click.STRING, required=True)
    100 
    +@pass_context
    101 
    +def wait(context, operation_name):
    102 
    +    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
    103 
    +    request = remote_execution_pb2.WaitExecutionRequest(instance_name=context.instance_name,
    104 
    +                                                        name=operation_name)
    105 
    +
    106 
    +    response = stub.WaitExecution(request)
    107 
    +
    108 
    +    for stream in response:
    109 
    +        context.logger.info(stream)

  • docs/source/data/buildstream-example-server.conf
    1 
    +server:
    2 
    +  port: 50051
    3 
    +  insecure-mode: true
    4 
    +
    5 
    +instances:
    6 
    +  - name: ""
    7 
    +
    8 
    +    storages:
    9 
    +      - !lru-storage &main-storage
    10 
    +        size: 512MB
    11 
    +
    12 
    +    services:
    13 
    +      - !action-cache &main-action
    14 
    +        storage: *main-storage
    15 
    +        max_cached_refs: 256
    16 
    +        allow_updates: true
    17 
    +      - !execution
    18 
    +        storage: *main-storage
    19 
    +        action_cache: *main-action
    20 
    +      - !cas
    21 
    +        storage: *main-storage
    22 
    +      - !reference-cache
    23 
    +        storage: *main-storage
    24 
    +        max_cached_refs: 128
    25 
    +      - !bytestream
    26 
    +        storage: *main-storage
    \ No newline at end of file

  • docs/source/reference_cli.rst
    1 1
    2 
    -.. _cli_reference:
    2 
    +.. _cli-reference:
    3 3
    4 4 
     CLI reference
    5 5 
     =============
    ... ... @@ -8,112 +8,118 @@ BuildGrid's Command Line Interface (CLI) reference documentation.
    8 8
    9 9 
     ----
    10 10
    11 
    -.. _invoking_bgd:
    11 
    +.. _invoking-bgd:
    12 12
    13 13 
     .. click:: buildgrid._app:cli
    14 14 
        :prog: bgd
    15 15
    16 16 
     ----
    17 17
    18 
    -.. _invoking_bgd_bot:
    18 
    +.. _invoking-bgd-bot:
    19 19
    20 20 
     .. click:: buildgrid._app.commands.cmd_bot:cli
    21 21 
        :prog: bgd bot
    22 22
    23 23 
     ----
    24 24
    25 
    -.. _invoking_bgd_bot_buildbox:
    25 
    +.. _invoking-bgd-bot-buildbox:
    26 26
    27 27 
     .. click:: buildgrid._app.commands.cmd_bot:run_buildbox
    28 28 
        :prog: bgd bot buildbot
    29 29
    30 30 
     ----
    31 31
    32 
    -.. _invoking_bgd_bot_dummy:
    32 
    +.. _invoking-bgd-bot-dummy:
    33 33
    34 34 
     .. click:: buildgrid._app.commands.cmd_bot:run_dummy
    35 35 
        :prog: bgd bot dummy
    36 36
    37 37 
     ----
    38 38
    39 
    -.. _invoking_bgd_bot_temp_directory:
    39 
    +.. _invoking-bgd-bot-temp-directory:
    40 40
    41 41 
     .. click:: buildgrid._app.commands.cmd_bot:run_temp_directory
    42 42 
        :prog: bgd bot temp-directory
    43 43
    44 44 
     ----
    45 45
    46 
    -.. _invoking_bgd_cas:
    46 
    +.. _invoking-bgd-cas:
    47 47
    48 48 
     .. click:: buildgrid._app.commands.cmd_cas:cli
    49 49 
        :prog: bgd cas
    50 50
    51 51 
     ----
    52 52
    53 
    -.. _invoking_bgd_cas_upload_dir:
    53 
    +.. _invoking-bgd-cas-upload-dir:
    54 54
    55 55 
     .. click:: buildgrid._app.commands.cmd_cas:upload_dir
    56 56 
        :prog: bgd cas upload-dir
    57 57
    58 58 
     ----
    59 59
    60 
    -.. _invoking_bgd_cas_upload_files:
    60 
    +.. _invoking-bgd-cas-upload-files:
    61 61
    62 62 
     .. click:: buildgrid._app.commands.cmd_cas:upload_files
    63 63 
        :prog: bgd cas upload-files
    64 64
    65 65 
     ----
    66 66
    67 
    -.. _invoking_bgd_execute:
    67 
    +.. _invoking-bgd-execute:
    68 68
    69 69 
     .. click:: buildgrid._app.commands.cmd_execute:cli
    70 70 
        :prog: bgd execute
    71 71
    72 72 
     ----
    73 73
    74 
    -.. _invoking_bgd_execute_command:
    74 
    +.. _invoking-bgd-execute-command:
    75 75
    76 76 
     .. click:: buildgrid._app.commands.cmd_execute:command
    77 77 
        :prog: bgd execute command
    78 78
    79 79 
     ----
    80 80
    81 
    -.. _invoking_bgd_execute_list:
    81 
    +.. _invoking-bgd-execute-request-dummy:
    82 82
    83 
    -.. click:: buildgrid._app.commands.cmd_execute:list_operations
    84 
    -   :prog: bgd execute list
    83 
    +.. click:: buildgrid._app.commands.cmd_execute:request_dummy
    84 
    +   :prog: bgd execute request-dummy
    85 85
    86 86 
     ----
    87 87
    88 
    -.. _invoking_bgd_execute_ request_dummy:
    88 
    +.. _invoking-bgd-operation:
    89 89
    90 
    -.. click:: buildgrid._app.commands.cmd_execute:request_dummy
    91 
    -   :prog: bgd execute request-dummy
    90 
    +.. click:: buildgrid._app.commands.cmd_operation:cli
    91 
    +   :prog: bgd operation
    92 92
    93 93 
     ----
    94 94
    95 
    -.. _invoking_bgd_execute_status:
    95 
    +.. _invoking-bgd-operation-list:
    96 96
    97 
    -.. click:: buildgrid._app.commands.cmd_execute:operation_status
    98 
    -   :prog: bgd execute status
    97 
    +.. click:: buildgrid._app.commands.cmd_operation:lists
    98 
    +   :prog: bgd operation list
    99 99
    100 100 
     ----
    101 101
    102 
    -.. _invoking_bgd_execute_wait:
    102 
    +.. _invoking-bgd-operation-status:
    103 103
    104 
    -.. click:: buildgrid._app.commands.cmd_execute:wait_execution
    105 
    -   :prog: bgd execute wait
    104 
    +.. click:: buildgrid._app.commands.cmd_operation:status
    105 
    +   :prog: bgd operation status
    106 106
    107 107 
     ----
    108 108
    109 
    -.. _invoking_bgd_server:
    109 
    +.. _invoking-bgd-operation-wait:
    110 
    +
    111 
    +.. click:: buildgrid._app.commands.cmd_operation:wait
    112 
    +   :prog: bgd operation wait
    113 
    +
    114 
    +----
    115 
    +.. _invoking-bgd-server:
    110 116
    111 117 
     .. click:: buildgrid._app.commands.cmd_server:cli
    112 118 
        :prog: bgd server
    113 119
    114 120 
     ----
    115 121
    116 
    -.. _invoking_bgd_server_start:
    122 
    +.. _invoking-bgd-server-start:
    117 123
    118 124 
     .. click:: buildgrid._app.commands.cmd_server:start
    119 125 
        :prog: bgd server start

  • docs/source/using.rst
    ... ... @@ -7,7 +7,8 @@ Using
    7 7 
     This section covers how to run an use the BuildGrid build service.
    8 8
    9 9 
     .. toctree::
    10 
    -   :maxdepth: 2
    10 
    +   :maxdepth: 3
    11 11
    12 12 
        using_internal.rst
    13 13 
        using_bazel.rst
    14 
    +   using_buildstream.rst

  • docs/source/using_bazel.rst
    1 1
    2 
    -.. _bazel-builds:
    2 
    +.. _bazel-client:
    3 3
    4 
    -Bazel builds
    4 
    +Bazel client
    5 5 
     ============
    6 6
    7 7 
     `Bazel`_ is a *“fast, scalable, multi-language and extensible build system”*
    ... ... @@ -18,7 +18,7 @@ Configuration
    18 18 
     -------------
    19 19
    20 20 
     Bazel accepts many options that can either be specified as command line
    21 
    -arguments when involking the ``bazel`` tool or stored in a `.bazelrc`_
    21 
    +arguments when invoking the ``bazel`` tool or stored in a `.bazelrc`_
    22 22 
     configuration file. In order to activate remote build execution, the
    23 23 
     ``bazel build`` subcommand needs to be given specific `build options`_,
    24 24 
     including:
    ... ... @@ -65,7 +65,7 @@ Next, change the current directory to the Bazel workspace root:
    65 65
    66 66 
        cd bazel-examples/cpp-tutorial/stage3
    67 67
    68 
    -.. note::
    68 
    +.. hint::
    69 69
    70 70 
        All the commands in the instructions below are expected to be executed from
    71 71 
        that root directory (the stage3 example workspace's root directory).
    ... ... @@ -80,7 +80,7 @@ below, paste it in a ``server.conf`` file in the root directory:
    80 80 
     This defines a single ``main`` server instance implementing a
    81 81 
     ``ContentAddressableStorage`` (CAS) + ``ByteStream`` service together with an
    82 82 
     ``Execution`` + ``ActionCache`` service, both using the same in-memory storage.
    83 
    -You can then start the BuildGrid server deamon using that configuration by
    83 
    +You can then start the BuildGrid server daemon using that configuration by
    84 84 
     running:
    85 85
    86 86 
     .. code-block:: sh
    ... ... @@ -97,8 +97,10 @@ has ``gcc`` installed, run:
    97 97 
        bgd bot --remote=http://localhost:50051 --parent=main temp-directory
    98 98
    99 99 
     The ``--remote`` option is used to specify the server location (running on the
    100 
    -same machine here, and listenning to port 50051). The ``--parent`` option is
    101 
    -used to specifiy the server instance you except the bot to be attached to.
    100 
    +same machine here, and listening to port 50051). The ``--parent`` option is used
    101 
    +to specify the server instance you except the bot to be attached to. Refer to
    102 
    +the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
    103 
    +line interface details.
    102 104
    103 105 
     The BuildGrid server is now ready to accept jobs and execute them. Bazel needs
    104 106 
     some :ref:`configuration <bazel-configuration>` in order to run remote builds.
    ... ... @@ -109,7 +111,7 @@ file in the root directory:
    109 111
    110 112 
        build --spawn_strategy=remote --genrule_strategy=remote --remote_executor=localhost:50051 --remote_instance_name=main
    111 113
    112 
    -This ativates Bazel's remote execution mode and points to the ``main`` remote
    114 
    +This activates Bazel's remote execution mode and points to the ``main`` remote
    113 115 
     execution server instance at ``localhost:50051``.
    114 116
    115 117 
     You can finally have Bazel to build the example workspace by running:
    ... ... @@ -118,7 +120,7 @@ You can finally have Bazel to build the example workspace by running:
    118 120
    119 121 
        bazel build //main:hello-world
    120 122
    121 
    -You can verify that the example has been successfully build by runnig the
    123 
    +You can verify that the example has been successfully build by running the
    122 124 
     generated executable. Simply invoke:
    123 125
    124 126 
     .. code-block:: sh

  • docs/source/using_buildstream.rst
    1 
    +
    2 
    +.. _buildstream-client:
    3 
    +
    4 
    +BuildStream client
    5 
    +==================
    6 
    +
    7 
    +`BuildStream`_ is a *“tool for integrating software stacks”*: it produces build
    8 
    +outputs from multiple input sources, using the existing sources build systems.
    9 
    +It supports remote build execution using the remote execution API (REAPI) v2.
    10 
    +
    11 
    +.. note::
    12 
    +
    13 
    +   No stable BuildStream version have been released with remote execution
    14 
    +   support yet: you'll have to `install it from sources`_ in order use remote
    15 
    +   build execution.
    16 
    +
    17 
    +.. _BuildStream: https://buildstream.build
    18 
    +.. _install it from sources: https://buildstream.build/source_install.html
    19 
    +
    20 
    +
    21 
    +.. _buildstream-configuration:
    22 
    +
    23 
    +Configuration
    24 
    +-------------
    25 
    +
    26 
    +BuildStream uses `YAML`_ for build definition and configuration. It has two
    27 
    +levels of configuration: user and project level. `User-level configuration`_ is
    28 
    +stored in your ``buildstream.conf`` file while `project-level configuration`_
    29 
    +is defined in each project's ``project.conf`` file.
    30 
    +
    31 
    +.. note::
    32 
    +
    33 
    +   At the moment, remote execution can only be configured at project-level.
    34 
    +
    35 
    +.. _YAML: http://yaml.org
    36 
    +.. _User-level configuration: https://buildstream.gitlab.io/buildstream/using_config.html
    37 
    +.. _project-level configuration: https://buildstream.gitlab.io/buildstream/format_project.html
    38 
    +
    39 
    +
    40 
    +Project configuration
    41 
    +~~~~~~~~~~~~~~~~~~~~~
    42 
    +
    43 
    +In order to activate remote build execution at project-level, the project's
    44 
    +``project.conf`` file must declare two specific configuration nodes:
    45 
    +
    46 
    +- ``artifacts`` for `remote CAS endpoint details`_.
    47 
    +- ``remote-execution`` for `remote execution endpoint details`_.
    48 
    +
    49 
    +.. important::
    50 
    +
    51 
    +   BuildStream does not support multi-instance remote execution servers and thus
    52 
    +   will always submit remote execution request omitting the instance name
    53 
    +   parameter.
    54 
    +
    55 
    +.. important::
    56 
    +
    57 
    +   If you are using BuildGrid's artifact server, the server instance **must**
    58 
    +   accept pushes from your client for remote execution to be possible.
    59 
    +
    60 
    +
    61 
    +.. _remote CAS endpoint details: https://buildstream.gitlab.io/buildstream/install_artifacts.html#user-configuration
    62 
    +.. _remote execution endpoint details: https://buildstream.gitlab.io/buildstream/format_project.html#remote-execution
    63 
    +
    64 
    +
    65 
    +BuildBox tool
    66 
    +~~~~~~~~~~~~~
    67 
    +
    68 
    +BuildStream performs builds in a `sandbox`_ on top of a project-defined
    69 
    +environment, not relying on any host-tools. BuildGrid supports this kind of
    70 
    +builds using the ``buildbox`` bot, a specific kind of bot relying on
    71 
    +`BuildBox`_ for build execution.
    72 
    +
    73 
    +BuildBox can execute build commands in a sandbox on top of an environment
    74 
    +constructed from provided sources using `FUSE`_. It also uses `bubblewrap`_
    75 
    +for sandboxing without requiring root privileges.
    76 
    +
    77 
    +BuildBox being a rather young project, it isn't packaged yet and you'll have to
    78 
    +build it from sources. You may want follow the `manual instructions`_ or prefer
    79 
    +to build it with BuildStream using the `dedicated integration project`_
    80 
    +(recommanded).
    81 
    +
    82 
    +.. important::
    83 
    +
    84 
    +   Whatever the method you use to install BuildBox, you also have to install
    85 
    +   bubblewrap along, minimum required version being `0.1.8`_.
    86 
    +
    87 
    +.. _sandbox: https://buildstream.gitlab.io/buildstream/additional_sandboxing.html
    88 
    +.. _BuildBox: https://gitlab.com/BuildStream/buildbox
    89 
    +.. _FUSE: https://en.wikipedia.org/wiki/Filesystem_in_Userspace
    90 
    +.. _bubblewrap: https://github.com/projectatomic/bubblewrap
    91 
    +.. _manual instructions: https://gitlab.com/BuildStream/buildbox/blob/master/INSTALL.rst
    92 
    +.. _dedicated integration project: https://gitlab.com/BuildStream/buildbox-integration
    93 
    +.. _0.1.8: https://github.com/projectatomic/bubblewrap/releases/tag/v0.1.8
    94 
    +
    95 
    +
    96 
    +.. _buildstream-example:
    97 
    +
    98 
    +Example build
    99 
    +-------------
    100 
    +
    101 
    +The BuildStream repository contains `example projects`_ used for testing purpose
    102 
    +in the project's `usage documentation section`_. We'll focus here on
    103 
    +instructions on how to build the `autotools example`_ running BuildStream and
    104 
    +BuildGrid on your local machine, compiling the `GNU Automake`_ hello example
    105 
    +program in a sandbox on top of a minimal `Alpine Linux`_ environment.
    106 
    +
    107 
    +First, you need to checkout the buildstream repository sources:
    108 
    +
    109 
    +.. code-block:: sh
    110 
    +
    111 
    +   git clone https://gitlab.com/BuildStream/buildstream.git
    112 
    +
    113 
    +Next, change the current directory to the BuildStream project root:
    114 
    +
    115 
    +.. code-block:: sh
    116 
    +
    117 
    +   cd buildstream/doc/examples/autotools
    118 
    +
    119 
    +.. hint::
    120 
    +
    121 
    +   All the commands in the instructions below are expected to be executed from
    122 
    +   that root directory (the autotools example project's root directory).
    123 
    +
    124 
    +Before running BuildStream and building the example project, you'll have to setup
    125 
    +and run a BuildGrid server and bot. A minimal server's configuration is given
    126 
    +below, paste it in a ``server.conf`` file in the root directory:
    127 
    +
    128 
    +.. literalinclude:: ./data/buildstream-example-server.conf
    129 
    +   :language: yaml
    130 
    +
    131 
    +This defines a single unnamed server instance implementing a
    132 
    +``ContentAddressableStorage`` (CAS) + ``Reference`` + ``ByteStream`` service
    133 
    +together with an ``Execution`` + ``ActionCache`` service, both using the
    134 
    +same in-memory storage. You can then start the BuildGrid server daemon using
    135 
    +that configuration by running:
    136 
    +
    137 
    +.. code-block:: sh
    138 
    +
    139 
    +   bgd server start server.conf
    140 
    +
    141 
    +In order to perform the actual build work, you need to attach a ``buildbox``
    142 
    +worker bot to that server for that unnamed instance. Once you've make sure
    143 
    +that the ``buildbox`` tool is functional on your machine (refer to
    144 
    +:ref:`configuration <buildstream-configuration>`), run:
    145 
    +
    146 
    +.. code-block:: sh
    147 
    +
    148 
    +   bgd bot --remote=http://localhost:50051 --parent= buildbox --fuse-dir=fuse --local-cas=cache
    149 
    +
    150 
    +The ``--remote`` option is used to specify the server location (running on the
    151 
    +same machine here, and listening to port 50051). The ``--parent`` option is
    152 
    +used to specify the server instance you except the bot to be attached to (empty
    153 
    +here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
    154 
    +and respectively specify the sandbox mount point and local CAS cache locations.
    155 
    +Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
    156 
    +for command line interface details.
    157 
    +
    158 
    +The BuildGrid server is now ready to accept jobs and execute them. The example
    159 
    +project needs some :ref:`configuration <buildstream-configuration>` tweaks in
    160 
    +order to be build remotely. Below is the configuration fragment you should
    161 
    +append at the end of the ``project.conf`` file from the root directory:
    162 
    +
    163 
    +.. code-block:: yaml
    164 
    +
    165 
    +   artifacts:
    166 
    +     url: http://localhost:50051
    167 
    +     push: true
    168 
    +
    169 
    +   remote-execution:
    170 
    +     url: http://localhost:50051
    171 
    +
    172 
    +This activates BuildGrid's remote execution mode and points to the unnamed
    173 
    +remote execution server instance at ``localhost:50051``.
    174 
    +
    175 
    +You can finally have BuildStream to build the example project by running:
    176 
    +
    177 
    +.. code-block:: sh
    178 
    +
    179 
    +   bst build hello.bst
    180 
    +
    181 
    +You can verify that the example has been successfully build by running the
    182 
    +generated executable. Simply invoke:
    183 
    +
    184 
    +.. code-block:: sh
    185 
    +
    186 
    +   bst shell hello.bst -- hello
    187 
    +
    188 
    +.. _example projects: https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples
    189 
    +.. _usage documentation section: http://buildstream.gitlab.io/buildstream/main_using.html
    190 
    +.. _autotools example: https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/autotools
    191 
    +.. _GNU Automake: https://www.gnu.org/software/automake
    192 
    +.. _Alpine Linux: https://www.alpinelinux.org

  • docs/source/using_internal.rst
    1 1
    2 
    -.. _internal-builds:
    2 
    +.. _internal-client:
    3 3
    4 
    -Internal builds
    4 
    +Internal client
    5 5 
     ===============
    6 6
    7 
    +BuildGrid contains a minimal remote execution client that can be used through
    8 
    +the ``bgd`` command line interface.
    9 
    +
    7 10
    8 11 
     .. _dummy-test:
    9 12

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]