Martin Blanchard pushed to branch mablanch/docker-compose at BuildGrid / buildgrid
Commits:
-
71de8e12
by Martin Blanchard at 2019-02-15T14:57:42Z
-
ced3e952
by Martin Blanchard at 2019-02-15T14:57:43Z
-
819c48fa
by Martin Blanchard at 2019-02-15T14:57:43Z
13 changed files:
- + .dockerignore
- .gitlab-ci.yml
- Dockerfile
- − buildgrid/_app/settings/default.yml
- − buildgrid/_app/settings/remote-storage.yml
- buildgrid/_app/settings/cas.yml → data/config/artifacts.conf
- + data/config/controller.conf
- + data/config/default.conf
- + data/config/storage.conf
- + docker-compose.yml
- docs/source/installation.rst
- docs/source/using_cas_server.rst
- docs/source/using_internal.rst
Changes:
| 1 |
+# Ignore everything:
|
|
| 2 |
+*
|
|
| 3 |
+ |
|
| 4 |
+# Whitelist:
|
|
| 5 |
+!buildgrid
|
|
| 6 |
+!tests
|
|
| 7 |
+!LICENSE
|
|
| 8 |
+!README.rst
|
|
| 9 |
+!setup.cfg
|
|
| 10 |
+!setup.py
|
|
| 11 |
+!_version.py
|
|
| 12 |
+!.coveragerc
|
|
| 13 |
+!.pylintrc
|
| ... | ... | @@ -31,7 +31,7 @@ before_script: |
| 31 | 31 |
.run-dummy-job-template: &dummy-job
|
| 32 | 32 |
stage: test
|
| 33 | 33 |
script:
|
| 34 |
- - ${BGD} server start buildgrid/_app/settings/default.yml &
|
|
| 34 |
+ - ${BGD} server start data/config/default.conf &
|
|
| 35 | 35 |
- sleep 1 # Allow server to boot
|
| 36 | 36 |
- ${BGD} bot dummy &
|
| 37 | 37 |
- ${BGD} cas upload-dummy
|
| 1 | 1 |
FROM python:3.5-stretch
|
| 2 | 2 |
|
| 3 |
-# Point the path to where buildgrid gets installed
|
|
| 4 |
-ENV PATH=$PATH:/root/.local/bin/
|
|
| 3 |
+# Use /app as working directory:
|
|
| 4 |
+WORKDIR /app
|
|
| 5 | 5 |
|
| 6 |
-# Upgrade python modules
|
|
| 7 |
-RUN python3 -m pip install --upgrade setuptools pip
|
|
| 6 |
+# Copy the repo. contents:
|
|
| 7 |
+COPY . /app
|
|
| 8 | 8 |
|
| 9 |
-# Use /app as the current working directory
|
|
| 10 |
-WORKDIR /app
|
|
| 9 |
+# Create a virtual environment:
|
|
| 10 |
+RUN python3 -m venv /app/env
|
|
| 11 | 11 |
|
| 12 |
-# Copy the repo contents (source, config files, etc) in the WORKDIR
|
|
| 13 |
-COPY . .
|
|
| 12 |
+# Upgrade Python core modules:
|
|
| 13 |
+RUN /app/env/bin/python -m pip install --upgrade setuptools pip wheel
|
|
| 14 | 14 |
|
| 15 |
-# Install BuildGrid
|
|
| 16 |
-RUN pip install --user --editable .
|
|
| 15 |
+# Install BuildGrid:
|
|
| 16 |
+RUN /app/env/bin/python -m pip install --editable ".[auth,tests]"
|
|
| 17 | 17 |
|
| 18 |
-# Entry Point of the image (should get an additional argument from CMD, the path to the config file)
|
|
| 19 |
-ENTRYPOINT ["bgd", "server", "start", "-vv"]
|
|
| 18 |
+# Entry-point for the image:
|
|
| 19 |
+ENTRYPOINT ["/app/env/bin/bgd"]
|
|
| 20 | 20 |
|
| 21 |
-# Default config file (used if no CMD specified when running)
|
|
| 22 |
-CMD ["buildgrid/_app/settings/default.yml"]
|
|
| 21 |
+# Default command (server with default config.):
|
|
| 22 |
+CMD ["server", "start", "data/config/default.conf", "-vvv"]
|
| 1 |
-server:
|
|
| 2 |
- - !channel
|
|
| 3 |
- port: 50051
|
|
| 4 |
- insecure-mode: true
|
|
| 5 |
- |
|
| 6 |
-description: |
|
|
| 7 |
- A single default instance.
|
|
| 8 |
- |
|
| 9 |
-instances:
|
|
| 10 |
- - name: ''
|
|
| 11 |
- description: |
|
|
| 12 |
- The main server
|
|
| 13 |
- |
|
| 14 |
- storages:
|
|
| 15 |
- - !disk-storage &main-storage
|
|
| 16 |
- path: !expand-path $HOME/cas
|
|
| 17 |
- |
|
| 18 |
- services:
|
|
| 19 |
- - !action-cache &main-action
|
|
| 20 |
- storage: *main-storage
|
|
| 21 |
- max-cached-refs: 256
|
|
| 22 |
- allow-updates: true
|
|
| 23 |
- |
|
| 24 |
- - !execution
|
|
| 25 |
- storage: *main-storage
|
|
| 26 |
- action-cache: *main-action
|
|
| 27 |
- |
|
| 28 |
- - !cas
|
|
| 29 |
- storage: *main-storage
|
|
| 30 |
- |
|
| 31 |
- - !bytestream
|
|
| 32 |
- storage: *main-storage
|
| 1 |
-server:
|
|
| 2 |
- - !channel
|
|
| 3 |
- port: 50051
|
|
| 4 |
- insecure-mode: true
|
|
| 5 |
- |
|
| 6 |
-description: |
|
|
| 7 |
- A single default instance with remote storage.
|
|
| 8 |
- |
|
| 9 |
-instances:
|
|
| 10 |
- - name: ''
|
|
| 11 |
- description: |
|
|
| 12 |
- The main server
|
|
| 13 |
- |
|
| 14 |
- storages:
|
|
| 15 |
- - !remote-storage &main-storage
|
|
| 16 |
- url: http://localhost:50052
|
|
| 17 |
- instance-name: main
|
|
| 18 |
- # credentials:
|
|
| 19 |
- # tls-client-key: null
|
|
| 20 |
- # tls-client-cert: null
|
|
| 21 |
- # tls-server-cert: null
|
|
| 22 |
- |
|
| 23 |
- services:
|
|
| 24 |
- - !action-cache &main-action
|
|
| 25 |
- storage: *main-storage
|
|
| 26 |
- max-cached-refs: 256
|
|
| 27 |
- allow-updates: true
|
|
| 28 |
- |
|
| 29 |
- - !execution
|
|
| 30 |
- storage: *main-storage
|
|
| 31 |
- action-cache: *main-action
|
|
| 32 |
- |
|
| 33 |
- - !cas
|
|
| 34 |
- storage: *main-storage
|
|
| 35 |
- |
|
| 36 |
- - !bytestream
|
|
| 37 |
- storage: *main-storage
|
| ... | ... | @@ -2,31 +2,34 @@ server: |
| 2 | 2 |
- !channel
|
| 3 | 3 |
port: 50052
|
| 4 | 4 |
insecure-mode: true
|
| 5 |
- # credentials:
|
|
| 6 |
- # tls-server-key: null
|
|
| 7 |
- # tls-server-cert: null
|
|
| 8 |
- # tls-client-certs: null
|
|
| 9 | 5 |
|
| 10 |
-description: |
|
|
| 11 |
- Just a CAS with some reference storage.
|
|
| 6 |
+description: >
|
|
| 7 |
+ Artifact server configuration:
|
|
| 8 |
+ - Unauthenticated plain HTTP at :50052
|
|
| 9 |
+ - Single instance: (empty-name)
|
|
| 10 |
+ - On-disk data stored in $HOME
|
|
| 11 |
+ - Hosted services:
|
|
| 12 |
+ - ReferenceStorage
|
|
| 13 |
+ - ContentAddressableStorage
|
|
| 14 |
+ - ByteStream
|
|
| 12 | 15 |
|
| 13 | 16 |
instances:
|
| 14 | 17 |
- name: ''
|
| 15 | 18 |
description: |
|
| 16 |
- The main server
|
|
| 19 |
+ The unique '' instance.
|
|
| 17 | 20 |
|
| 18 | 21 |
storages:
|
| 19 |
- - !disk-storage &main-storage
|
|
| 20 |
- path: !expand-path $HOME/cas
|
|
| 22 |
+ - !disk-storage &data-store
|
|
| 23 |
+ path: !expand-path $HOME/.cache/buildgrid/store
|
|
| 21 | 24 |
|
| 22 | 25 |
services:
|
| 23 | 26 |
- !cas
|
| 24 |
- storage: *main-storage
|
|
| 27 |
+ storage: *data-store
|
|
| 25 | 28 |
|
| 26 | 29 |
- !bytestream
|
| 27 |
- storage: *main-storage
|
|
| 30 |
+ storage: *data-store
|
|
| 28 | 31 |
|
| 29 | 32 |
- !reference-cache
|
| 30 |
- storage: *main-storage
|
|
| 31 |
- max-cached-refs: 256
|
|
| 33 |
+ storage: *data-store
|
|
| 34 |
+ max-cached-refs: 512
|
|
| 32 | 35 |
allow-updates: true
|
| 1 |
+server:
|
|
| 2 |
+ - !channel
|
|
| 3 |
+ port: 50051
|
|
| 4 |
+ insecure-mode: true
|
|
| 5 |
+ |
|
| 6 |
+description: >
|
|
| 7 |
+ Docker Compose controller configuration:
|
|
| 8 |
+ - Unauthenticated plain HTTP at :50051
|
|
| 9 |
+ - Single instance: local
|
|
| 10 |
+ - Expects a remote CAS at :50052
|
|
| 11 |
+ - Hosted services:
|
|
| 12 |
+ - ActionCache
|
|
| 13 |
+ - Execute
|
|
| 14 |
+ |
|
| 15 |
+authorization:
|
|
| 16 |
+ method: none
|
|
| 17 |
+ |
|
| 18 |
+monitoring:
|
|
| 19 |
+ enabled: false
|
|
| 20 |
+ |
|
| 21 |
+instances:
|
|
| 22 |
+ - name: local
|
|
| 23 |
+ description: |
|
|
| 24 |
+ The unique 'local' instance.
|
|
| 25 |
+ |
|
| 26 |
+ storages:
|
|
| 27 |
+ - !remote-storage &data-store
|
|
| 28 |
+ url: http://storage:50052
|
|
| 29 |
+ instance-name: local
|
|
| 30 |
+ |
|
| 31 |
+ services:
|
|
| 32 |
+ - !action-cache &build-cache
|
|
| 33 |
+ storage: *data-store
|
|
| 34 |
+ max-cached-refs: 256
|
|
| 35 |
+ cache-failed-actions: true
|
|
| 36 |
+ allow-updates: true
|
|
| 37 |
+ |
|
| 38 |
+ - !execution
|
|
| 39 |
+ storage: *data-store
|
|
| 40 |
+ action-cache: *build-cache
|
| 1 |
+server:
|
|
| 2 |
+ - !channel
|
|
| 3 |
+ port: 50051
|
|
| 4 |
+ insecure-mode: true
|
|
| 5 |
+ |
|
| 6 |
+description: >
|
|
| 7 |
+ BuildGrid's default configuration:
|
|
| 8 |
+ - Unauthenticated plain HTTP at :50052
|
|
| 9 |
+ - Single instance: main
|
|
| 10 |
+ - In-memory data, max. 2Gio
|
|
| 11 |
+ - Hosted services:
|
|
| 12 |
+ - ActionCache
|
|
| 13 |
+ - Execute
|
|
| 14 |
+ - ContentAddressableStorage
|
|
| 15 |
+ - ByteStream
|
|
| 16 |
+ |
|
| 17 |
+authorization:
|
|
| 18 |
+ method: none
|
|
| 19 |
+ |
|
| 20 |
+monitoring:
|
|
| 21 |
+ enabled: false
|
|
| 22 |
+ |
|
| 23 |
+instances:
|
|
| 24 |
+ - name: ''
|
|
| 25 |
+ description: |
|
|
| 26 |
+ The unique '' instance.
|
|
| 27 |
+ |
|
| 28 |
+ storages:
|
|
| 29 |
+ - !lru-storage &data-store
|
|
| 30 |
+ size: 2048M
|
|
| 31 |
+ |
|
| 32 |
+ services:
|
|
| 33 |
+ - !action-cache &build-cache
|
|
| 34 |
+ storage: *data-store
|
|
| 35 |
+ max-cached-refs: 256
|
|
| 36 |
+ cache-failed-actions: true
|
|
| 37 |
+ allow-updates: true
|
|
| 38 |
+ |
|
| 39 |
+ - !execution
|
|
| 40 |
+ storage: *data-store
|
|
| 41 |
+ action-cache: *build-cache
|
|
| 42 |
+ |
|
| 43 |
+ - !cas
|
|
| 44 |
+ storage: *data-store
|
|
| 45 |
+ |
|
| 46 |
+ - !bytestream
|
|
| 47 |
+ storage: *data-store
|
| 1 |
+server:
|
|
| 2 |
+ - !channel
|
|
| 3 |
+ port: 50052
|
|
| 4 |
+ insecure-mode: true
|
|
| 5 |
+ |
|
| 6 |
+description: >
|
|
| 7 |
+ Docker Compose storage configuration:
|
|
| 8 |
+ - Unauthenticated plain HTTP at :50052
|
|
| 9 |
+ - Single instance: local
|
|
| 10 |
+ - On-disk data stored in /var
|
|
| 11 |
+ - Hosted services:
|
|
| 12 |
+ - ReferenceStorage
|
|
| 13 |
+ - ContentAddressableStorage
|
|
| 14 |
+ - ByteStream
|
|
| 15 |
+ |
|
| 16 |
+authorization:
|
|
| 17 |
+ method: none
|
|
| 18 |
+ |
|
| 19 |
+monitoring:
|
|
| 20 |
+ enabled: false
|
|
| 21 |
+ |
|
| 22 |
+instances:
|
|
| 23 |
+ - name: local
|
|
| 24 |
+ description: |
|
|
| 25 |
+ The unique 'local' instance.
|
|
| 26 |
+ |
|
| 27 |
+ storages:
|
|
| 28 |
+ - !disk-storage &data-store
|
|
| 29 |
+ path: /var/lib/buildgrid/store
|
|
| 30 |
+ |
|
| 31 |
+ services:
|
|
| 32 |
+ - !cas
|
|
| 33 |
+ storage: *data-store
|
|
| 34 |
+ |
|
| 35 |
+ - !bytestream
|
|
| 36 |
+ storage: *data-store
|
|
| 37 |
+ |
|
| 38 |
+ - !reference-cache
|
|
| 39 |
+ storage: *data-store
|
|
| 40 |
+ max-cached-refs: 1024
|
|
| 41 |
+ allow-updates: true
|
| 1 |
+##
|
|
| 2 |
+# BuildGrid's Docker Compose manifest.
|
|
| 3 |
+#
|
|
| 4 |
+# ¡FOR LOCAL TESTING ONLY!
|
|
| 5 |
+#
|
|
| 6 |
+# Spins-up a 'local' grid instance:
|
|
| 7 |
+# - Controller at http://localhost:50051
|
|
| 8 |
+# - CAS + AC at: http://localhost:50052
|
|
| 9 |
+#
|
|
| 10 |
+# Basic usage:
|
|
| 11 |
+# - docker-compose build
|
|
| 12 |
+# - docker-compose up --scale bots=10
|
|
| 13 |
+# - docker-compose down
|
|
| 14 |
+# - docker volume inspect buildgrid_data
|
|
| 15 |
+# - docker volume rm buildgrid_data
|
|
| 16 |
+#
|
|
| 17 |
+version: "3.2"
|
|
| 18 |
+ |
|
| 19 |
+services:
|
|
| 20 |
+ storage:
|
|
| 21 |
+ build:
|
|
| 22 |
+ context: .
|
|
| 23 |
+ image: buildgrid:local
|
|
| 24 |
+ command: [
|
|
| 25 |
+ "server", "start", "-vvv",
|
|
| 26 |
+ "/app/config/storage.conf"]
|
|
| 27 |
+ volumes:
|
|
| 28 |
+ - type: volume
|
|
| 29 |
+ source: data
|
|
| 30 |
+ target: /var/lib/buildgrid/store
|
|
| 31 |
+ volume:
|
|
| 32 |
+ nocopy: true
|
|
| 33 |
+ - type: bind
|
|
| 34 |
+ source: ./data/config/storage.conf
|
|
| 35 |
+ target: /app/config/storage.conf
|
|
| 36 |
+ ports:
|
|
| 37 |
+ - "50052:50052"
|
|
| 38 |
+ networks:
|
|
| 39 |
+ - backend
|
|
| 40 |
+ - host
|
|
| 41 |
+ |
|
| 42 |
+ controller:
|
|
| 43 |
+ image: buildgrid:local
|
|
| 44 |
+ command: [
|
|
| 45 |
+ "server", "start", "-vvv",
|
|
| 46 |
+ "/app/config/controller.conf"]
|
|
| 47 |
+ volumes:
|
|
| 48 |
+ - type: bind
|
|
| 49 |
+ source: ./data/config/controller.conf
|
|
| 50 |
+ target: /app/config/controller.conf
|
|
| 51 |
+ ports:
|
|
| 52 |
+ - "50051:50051"
|
|
| 53 |
+ networks:
|
|
| 54 |
+ - backend
|
|
| 55 |
+ - host
|
|
| 56 |
+ |
|
| 57 |
+ bots:
|
|
| 58 |
+ image: buildgrid:local
|
|
| 59 |
+ command: [
|
|
| 60 |
+ "bot", "--parent=local",
|
|
| 61 |
+ "--remote=http://controller:50051",
|
|
| 62 |
+ "--remote-cas=http://storage:50052",
|
|
| 63 |
+ "host-tools"]
|
|
| 64 |
+ depends_on:
|
|
| 65 |
+ - controller
|
|
| 66 |
+ networks:
|
|
| 67 |
+ - backend
|
|
| 68 |
+ |
|
| 69 |
+networks:
|
|
| 70 |
+ backend:
|
|
| 71 |
+ host:
|
|
| 72 |
+ |
|
| 73 |
+volumes:
|
|
| 74 |
+ data:
|
| ... | ... | @@ -110,7 +110,7 @@ To clone the source code and build a Docker image, simply run: |
| 110 | 110 |
the image.
|
| 111 | 111 |
|
| 112 | 112 |
After building the Docker image, to run BuildGrid using the default configuration file
|
| 113 |
-(found in `buildgrid/_app/settings/default.yml`), simply run:
|
|
| 113 |
+(found in `data/config/default.conf`), simply run:
|
|
| 114 | 114 |
|
| 115 | 115 |
.. code-block:: sh
|
| 116 | 116 |
|
| ... | ... | @@ -27,7 +27,7 @@ This defines a single ``main`` instance of the ``CAS``, ``Bytestream`` and ``Ref |
| 27 | 27 |
|
| 28 | 28 |
.. code-block:: sh
|
| 29 | 29 |
|
| 30 |
- bgd server start example.conf
|
|
| 30 |
+ bgd server start data/config/artifacts.conf
|
|
| 31 | 31 |
|
| 32 | 32 |
The server should now be available to use.
|
| 33 | 33 |
|
| ... | ... | @@ -17,7 +17,7 @@ In one terminal, start a server: |
| 17 | 17 |
|
| 18 | 18 |
.. code-block:: sh
|
| 19 | 19 |
|
| 20 |
- bgd server start buildgrid/_app/settings/default.yml
|
|
| 20 |
+ bgd server start data/config/default.conf
|
|
| 21 | 21 |
|
| 22 | 22 |
In another terminal, upload an action to CAS:
|
| 23 | 23 |
|
| ... | ... | @@ -78,7 +78,7 @@ Now start a BuildGrid server, passing it a directory it can write a CAS to: |
| 78 | 78 |
|
| 79 | 79 |
.. code-block:: sh
|
| 80 | 80 |
|
| 81 |
- bgd server start buildgrid/_app/settings/default.yml
|
|
| 81 |
+ bgd server start data/config/default.conf
|
|
| 82 | 82 |
|
| 83 | 83 |
Start the following bot session:
|
| 84 | 84 |
|