GNOME.org
 

[Notes] [Git][BuildStream/buildstream][tpollard/752] 10 commits: _project.py: Validate nodes early in Project._load

Title: GitLab

Tom Pollard pushed to branch tpollard/752 at BuildStream / buildstream

Commits:

23 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -79,32 +79,46 @@ source_dist:
    79 79 
       - cd ../..
    80 80 
       - mkdir -p coverage-linux/
    81 81 
       - cp dist/buildstream/.coverage coverage-linux/coverage."${CI_JOB_NAME}"
    82 
    -  except:
    83 
    -  - schedules
    84 82 
       artifacts:
    85 83 
         paths:
    86 84 
         - coverage-linux/
    87 85
    88 86 
     tests-debian-9:
    89 
    -  image: buildstream/testsuite-debian:9-master-119-552f5fc6
    87 
    +  image: buildstream/testsuite-debian:9-master-123-7ce6581b
    90 88 
       <<: *linux-tests
    89 
    +  except:
    90 
    +  - schedules
    91 91
    92 92 
     tests-fedora-27:
    93 
    -  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
    93 
    +  image: buildstream/testsuite-fedora:27-master-123-7ce6581b
    94 94 
       <<: *linux-tests
    95 
    +  except:
    96 
    +  - schedules
    95 97
    96 98 
     tests-fedora-28:
    97 
    -  image: buildstream/testsuite-fedora:28-master-119-552f5fc6
    99 
    +  image: buildstream/testsuite-fedora:28-master-123-7ce6581b
    98 100 
       <<: *linux-tests
    101 
    +  except:
    102 
    +  - schedules
    99 103
    100 104 
     tests-ubuntu-18.04:
    101 
    -  image: buildstream/testsuite-ubuntu:18.04-master-119-552f5fc6
    105 
    +  image: buildstream/testsuite-ubuntu:18.04-master-123-7ce6581b
    102 106 
       <<: *linux-tests
    107 
    +  except:
    108 
    +  - schedules
    109 
    +
    110 
    +overnight-fedora-28-aarch64:
    111 
    +  image: buildstream/testsuite-fedora:aarch64-28-master-123-7ce6581b
    112 
    +  tags:
    113 
    +    - aarch64
    114 
    +  <<: *linux-tests
    115 
    +  only:
    116 
    +  - schedules
    103 117
    104 118 
     tests-unix:
    105 119 
       # Use fedora here, to a) run a test on fedora and b) ensure that we
    106 120 
       # can get rid of ostree - this is not possible with debian-8
    107 
    -  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
    121 
    +  image: buildstream/testsuite-fedora:27-master-123-7ce6581b
    108 122 
       stage: test
    109 123 
       variables:
    110 124 
         BST_FORCE_BACKEND: "unix"

  • NEWS
    ... ... @@ -45,6 +45,15 @@ buildstream 1.3.1
    45 45 
         instead of just a specially-formatted build-root with a `root` and `scratch`
    46 46 
         subdirectory.
    47 47
    48 
    +  o bst interaction with defined artifact servers can be controlled more granularly.
    49 
    +    This can be done via the user configuration option `useremotes` or via the bst cli
    50 
    +    main option '--use-remotes'. This can be set as 'none', 'user' or the default value
    51 
    +    'all'. Unless specifically overriden, when considering wether to pull or push to
    52 
    +    available artifact servers (be it user or project defined) this optional config option
    53 
    +    will be used. Setting this value to 'user' for example and performing a build would
    54 
    +    lead to any project or junction defined artifact server to be ignored, whilst still
    55 
    +    attempting to any user defined remotes.
    56 
    +
    48 57
    49 58 
     =================
    50 59 
     buildstream 1.1.5

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -156,7 +156,7 @@ class ArtifactCache():
    156 156 
         # Sets up which remotes to use
    157 157 
         #
    158 158 
         # Args:
    159 
    -    #    use_config (bool): Whether to use project configuration
    159 
    +    #    use_config (bool): Whether to use configuration
    160 160 
         #    remote_url (str): Remote artifact cache URL
    161 161 
         #
    162 162 
         # This requires that all of the projects which are to be processed in the session
    ... ... @@ -175,11 +175,16 @@ class ArtifactCache():
    175 175 
                 self._set_remotes([ArtifactCacheSpec(remote_url, push=True)])
    176 176 
                 has_remote_caches = True
    177 177 
             if use_config:
    178 
    -            for project in self.context.get_projects():
    179 
    -                artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
    180 
    -                if artifact_caches:  # artifact_caches is a list of ArtifactCacheSpec instances
    181 
    -                    self._set_remotes(artifact_caches, project=project)
    182 
    -                    has_remote_caches = True
    178 
    +            if self.context.use_remotes == 'all':
    179 
    +                for project in self.context.get_projects():
    180 
    +                    artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
    181 
    +                    if artifact_caches:  # artifact_caches is a list of ArtifactCacheSpec instances
    182 
    +                        self._set_remotes(artifact_caches, project=project)
    183 
    +                        has_remote_caches = True
    184 
    +            # If configured to only use user configured remotes, pass existing user cache spec
    185 
    +            elif self.context.use_remotes == 'user' and self.context.artifact_cache_specs:
    186 
    +                self._set_remotes(self.context.artifact_cache_specs)
    187 
    +                has_remote_caches = True
    183 188 
             if has_remote_caches:
    184 189 
                 self._initialize_remotes()
    185 190

  • buildstream/_context.py
    ... ... @@ -110,6 +110,9 @@ class Context():
    110 110 
             # Make sure the XDG vars are set in the environment before loading anything
    111 111 
             self._init_xdg()
    112 112
    113 
    +        # Which remote artifact servers to interact with. all, user or none
    114 
    +        self.use_remotes = 'all'
    115 
    +
    113 116 
             # Private variables
    114 117 
             self._cache_key = None
    115 118 
             self._message_handler = None
    ... ... @@ -160,7 +163,7 @@ class Context():
    160 163 
             _yaml.node_validate(defaults, [
    161 164 
                 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    162 165 
                 'scheduler', 'artifacts', 'logging', 'projects',
    163 
    -            'cache'
    166 
    +            'cache', 'useremotes'
    164 167 
             ])
    165 168
    166 169 
             for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
    ... ... @@ -185,6 +188,13 @@ class Context():
    185 188 
             # Load artifact share configuration
    186 189 
             self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
    187 190
    191 
    +        # Load remote artifact server usage
    192 
    +        self.use_remotes = _yaml.node_get(defaults, str, 'useremotes', default_value='all')
    193 
    +        valid_actions = ['all', 'user', 'none']
    194 
    +        if self.use_remotes not in valid_actions:
    195 
    +            raise LoadError(LoadErrorReason.INVALID_DATA,
    196 
    +                            "useremotes should be one of: {}".format(", ".join(valid_actions)))
    197 
    +
    188 198 
             # Load logging config
    189 199 
             logging = _yaml.node_get(defaults, Mapping, 'logging')
    190 200 
             _yaml.node_validate(logging, [

  • buildstream/_frontend/app.py
    ... ... @@ -182,7 +182,8 @@ class App():
    182 182 
                 'fetchers': 'sched_fetchers',
    183 183 
                 'builders': 'sched_builders',
    184 184 
                 'pushers': 'sched_pushers',
    185 
    -            'network_retries': 'sched_network_retries'
    185 
    +            'network_retries': 'sched_network_retries',
    186 
    +            'use_remotes': 'use_remotes'
    186 187 
             }
    187 188 
             for cli_option, context_attr in override_map.items():
    188 189 
                 option_value = self._main_options.get(cli_option)

  • buildstream/_frontend/cli.py
    ... ... @@ -219,6 +219,9 @@ def print_version(ctx, param, value):
    219 219 
                   help="Specify a project option")
    220 220 
     @click.option('--default-mirror', default=None,
    221 221 
                   help="The mirror to fetch from first, before attempting other mirrors")
    222 
    +@click.option('--use-remotes', default='all',
    223 
    +              type=click.Choice(['all', 'user', 'none']),
    224 
    +              help='The remote artifact caches to interact with (default: all)')
    222 225 
     @click.pass_context
    223 226 
     def cli(context, **kwargs):
    224 227 
         """Build and manipulate BuildStream projects

  • buildstream/_project.py
    ... ... @@ -219,6 +219,19 @@ class Project():
    219 219
    220 220 
             return self._cache_key
    221 221
    222 
    +    def _validate_node(self, node):
    223 
    +        _yaml.node_validate(node, [
    224 
    +            'format-version',
    225 
    +            'element-path', 'variables',
    226 
    +            'environment', 'environment-nocache',
    227 
    +            'split-rules', 'elements', 'plugins',
    228 
    +            'aliases', 'name',
    229 
    +            'artifacts', 'options',
    230 
    +            'fail-on-overlap', 'shell', 'fatal-warnings',
    231 
    +            'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
    232 
    +            'sources', '(@)'
    233 
    +        ])
    234 
    +
    222 235 
         # create_element()
    223 236 
         #
    224 237 
         # Instantiate and return an element
    ... ... @@ -402,6 +415,8 @@ class Project():
    402 415 
                     "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
    403 416 
                     .format(format_version, major, minor, BST_FORMAT_VERSION))
    404 417
    418 
    +        self._validate_node(pre_config_node)
    419 
    +
    405 420 
             # FIXME:
    406 421 
             #
    407 422 
             #   Performing this check manually in the absense
    ... ... @@ -467,16 +482,7 @@ class Project():
    467 482
    468 483 
             self._load_pass(config, self.config)
    469 484
    470 
    -        _yaml.node_validate(config, [
    471 
    -            'format-version',
    472 
    -            'element-path', 'variables',
    473 
    -            'environment', 'environment-nocache',
    474 
    -            'split-rules', 'elements', 'plugins',
    475 
    -            'aliases', 'name',
    476 
    -            'artifacts', 'options',
    477 
    -            'fail-on-overlap', 'shell', 'fatal-warnings',
    478 
    -            'ref-storage', 'sandbox', 'mirrors', 'remote-execution'
    479 
    -        ])
    485 
    +        self._validate_node(config)
    480 486
    481 487 
             #
    482 488 
             # Now all YAML composition is done, from here on we just load

  • tests/cachekey/cachekey.py
    ... ... @@ -36,7 +36,7 @@
    36 36 
     # the result.
    37 37 
     #
    38 38 
     from tests.testutils.runcli import cli
    39 
    -from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX
    39 
    +from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX, MACHINE_ARCH
    40 40 
     from buildstream.plugin import CoreWarnings
    41 41 
     from buildstream import _yaml
    42 42 
     import os
    ... ... @@ -144,6 +144,8 @@ DATA_DIR = os.path.join(
    144 144 
     # The cache key test uses a project which exercises all plugins,
    145 145 
     # so we cant run it at all if we dont have them installed.
    146 146 
     #
    147 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    148 
    +                    reason='Cache keys depend on architecture')
    147 149 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    148 150 
     @pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available")
    149 151 
     @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")

  • tests/completions/completions.py
    ... ... @@ -44,6 +44,7 @@ MAIN_OPTIONS = [
    44 44 
         "--on-error ",
    45 45 
         "--pushers ",
    46 46 
         "--strict ",
    47 
    +    "--use-remotes ",
    47 48 
         "--verbose ",
    48 49 
         "--version ",
    49 50 
     ]
    ... ... @@ -117,6 +118,7 @@ def test_options(cli, cmd, word_idx, expected):
    117 118
    118 119 
     @pytest.mark.parametrize("cmd,word_idx,expected", [
    119 120 
         ('bst --on-error ', 2, ['continue ', 'quit ', 'terminate ']),
    121 
    +    ('bst --use-remotes ', 2, ['all ', 'user ', 'none ']),
    120 122 
         ('bst show --deps ', 3, ['all ', 'build ', 'none ', 'plan ', 'run ']),
    121 123 
         ('bst show --deps=', 2, ['all ', 'build ', 'none ', 'plan ', 'run ']),
    122 124 
         ('bst show --deps b', 3, ['build ']),

  • tests/examples/autotools.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8 
     pytestmark = pytest.mark.integration
    9 9
    ... ... @@ -13,6 +13,8 @@ DATA_DIR = os.path.join(
    13 13
    14 14
    15 15 
     # Tests a build of the autotools amhello project on a alpine-linux base runtime
    16 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    17 
    +                    reason='Examples are writtent for x86_64')
    16 18 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    17 19 
     @pytest.mark.datafiles(DATA_DIR)
    18 20 
     def test_autotools_build(cli, tmpdir, datafiles):
    ... ... @@ -36,6 +38,8 @@ def test_autotools_build(cli, tmpdir, datafiles):
    36 38
    37 39
    38 40 
     # Test running an executable built with autotools.
    41 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    42 
    +                    reason='Examples are writtent for x86_64')
    39 43 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    40 44 
     @pytest.mark.datafiles(DATA_DIR)
    41 45 
     def test_autotools_run(cli, tmpdir, datafiles):

  • tests/examples/developing.py
    ... ... @@ -4,7 +4,7 @@ import pytest
    4 4 
     import tests.testutils.patch as patch
    5 5 
     from tests.testutils import cli_integration as cli
    6 6 
     from tests.testutils.integration import assert_contains
    7 
    -from tests.testutils.site import IS_LINUX
    7 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    8 8
    9 9 
     pytestmark = pytest.mark.integration
    10 10
    ... ... @@ -14,6 +14,8 @@ DATA_DIR = os.path.join(
    14 14
    15 15
    16 16 
     # Test that the project builds successfully
    17 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    18 
    +                    reason='Examples are writtent for x86_64')
    17 19 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    18 20 
     @pytest.mark.datafiles(DATA_DIR)
    19 21 
     def test_autotools_build(cli, tmpdir, datafiles):
    ... ... @@ -35,6 +37,8 @@ def test_autotools_build(cli, tmpdir, datafiles):
    35 37
    36 38
    37 39 
     # Test the unmodified hello command works as expected.
    40 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    41 
    +                    reason='Examples are writtent for x86_64')
    38 42 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    39 43 
     @pytest.mark.datafiles(DATA_DIR)
    40 44 
     def test_run_unmodified_hello(cli, tmpdir, datafiles):
    ... ... @@ -66,6 +70,8 @@ def test_open_workspace(cli, tmpdir, datafiles):
    66 70
    67 71
    68 72 
     # Test making a change using the workspace
    73 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    74 
    +                    reason='Examples are writtent for x86_64')
    69 75 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    70 76 
     @pytest.mark.datafiles(DATA_DIR)
    71 77 
     def test_make_change_in_workspace(cli, tmpdir, datafiles):

  • tests/examples/flatpak-autotools.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8
    9 9 
     pytestmark = pytest.mark.integration
    ... ... @@ -32,6 +32,8 @@ def workaround_setuptools_bug(project):
    32 32
    33 33 
     # Test that a build upon flatpak runtime 'works' - we use the autotools sample
    34 34 
     # amhello project for this.
    35 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    36 
    +                    reason='Examples are writtent for x86_64')
    35 37 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    36 38 
     @pytest.mark.datafiles(DATA_DIR)
    37 39 
     def test_autotools_build(cli, tmpdir, datafiles):
    ... ... @@ -55,6 +57,8 @@ def test_autotools_build(cli, tmpdir, datafiles):
    55 57
    56 58
    57 59 
     # Test running an executable built with autotools
    60 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    61 
    +                    reason='Examples are writtent for x86_64')
    58 62 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    59 63 
     @pytest.mark.datafiles(DATA_DIR)
    60 64 
     def test_autotools_run(cli, tmpdir, datafiles):

  • tests/examples/integration-commands.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8
    9 9 
     pytestmark = pytest.mark.integration
    ... ... @@ -12,6 +12,8 @@ DATA_DIR = os.path.join(
    12 12 
     )
    13 13
    14 14
    15 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    16 
    +                    reason='Examples are writtent for x86_64')
    15 17 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    16 18 
     @pytest.mark.datafiles(DATA_DIR)
    17 19 
     def test_integration_commands_build(cli, tmpdir, datafiles):
    ... ... @@ -23,6 +25,8 @@ def test_integration_commands_build(cli, tmpdir, datafiles):
    23 25
    24 26
    25 27 
     # Test running the executable
    28 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    29 
    +                    reason='Examples are writtent for x86_64')
    26 30 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    27 31 
     @pytest.mark.datafiles(DATA_DIR)
    28 32 
     def test_integration_commands_run(cli, tmpdir, datafiles):

  • tests/examples/junctions.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8 
     pytestmark = pytest.mark.integration
    9 9
    ... ... @@ -13,6 +13,8 @@ DATA_DIR = os.path.join(
    13 13
    14 14
    15 15 
     # Test that the project builds successfully
    16 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    17 
    +                    reason='Examples are writtent for x86_64')
    16 18 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    17 19 
     @pytest.mark.datafiles(DATA_DIR)
    18 20 
     def test_build(cli, tmpdir, datafiles):
    ... ... @@ -23,6 +25,8 @@ def test_build(cli, tmpdir, datafiles):
    23 25
    24 26
    25 27 
     # Test the callHello script works as expected.
    28 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    29 
    +                    reason='Examples are writtent for x86_64')
    26 30 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    27 31 
     @pytest.mark.datafiles(DATA_DIR)
    28 32 
     def test_shell_call_hello(cli, tmpdir, datafiles):

  • tests/examples/running-commands.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8
    9 9 
     pytestmark = pytest.mark.integration
    ... ... @@ -12,6 +12,8 @@ DATA_DIR = os.path.join(
    12 12 
     )
    13 13
    14 14
    15 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    16 
    +                    reason='Examples are writtent for x86_64')
    15 17 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    16 18 
     @pytest.mark.datafiles(DATA_DIR)
    17 19 
     def test_running_commands_build(cli, tmpdir, datafiles):
    ... ... @@ -23,6 +25,8 @@ def test_running_commands_build(cli, tmpdir, datafiles):
    23 25
    24 26
    25 27 
     # Test running the executable
    28 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    29 
    +                    reason='Examples are writtent for x86_64')
    26 30 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    27 31 
     @pytest.mark.datafiles(DATA_DIR)
    28 32 
     def test_running_commands_run(cli, tmpdir, datafiles):

  • tests/format/list-directive-type-error/project.conf
    ... ... @@ -4,4 +4,4 @@ options:
    4 4 
       arch:
    5 5 
         type: arch
    6 6 
         description: Example architecture option
    7 
    -    values: [ x86_32, x86_64 ]
    7 
    +    values: [ x86_32, x86_64, aarch64 ]

  • tests/frontend/invalid_element_path/project.conf
    1 
    +# Project config for frontend build test
    2 
    +name: test
    3 
    +
    4 
    +elephant-path: elements

  • tests/frontend/pull.py
    ... ... @@ -358,3 +358,75 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
    358 358
    359 359 
             assert "INFO    Remote ({}) does not have".format(share.repo) in result.stderr
    360 360 
             assert "SKIPPED Pull" in result.stderr
    361 
    +
    362 
    +
    363 
    +# Tests that:
    364 
    +#
    365 
    +#  * The bst main option --use-remotes limits remote action
    366 
    +#    as expected for pull jobs
    367 
    +#
    368 
    +@pytest.mark.datafiles(DATA_DIR)
    369 
    +def test_useremotes_cli_options(cli, tmpdir, datafiles):
    370 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    371 
    +
    372 
    +    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
    373 
    +        create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
    374 
    +
    375 
    +        # Add shareproject repo url to project.conf
    376 
    +        with open(os.path.join(project, "project.conf"), "a") as projconf:
    377 
    +            projconf.write("artifacts:\n  url: {}\n  push: True".format(shareproject.repo))
    378 
    +
    379 
    +        # First build the target element and push to the remotes.
    380 
    +        # We need the artifact available in the remotes to test against.
    381 
    +        cli.configure({
    382 
    +            'artifacts': {'url': shareuser.repo, 'push': True}
    383 
    +        })
    384 
    +        result = cli.run(project=project, args=['build', 'target.bst'])
    385 
    +        result.assert_success()
    386 
    +        assert cli.get_element_state(project, 'target.bst') == 'cached'
    387 
    +
    388 
    +        # Assert that everything is now cached in the remotes.
    389 
    +        all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
    390 
    +        for element_name in all_elements:
    391 
    +            assert_shared(cli, shareuser, project, element_name)
    392 
    +            assert_shared(cli, shareproject, project, element_name)
    393 
    +
    394 
    +        # Now we've pushed, delete the user's local artifact cache
    395 
    +        artifacts = os.path.join(cli.directory, 'artifacts')
    396 
    +        shutil.rmtree(artifacts)
    397 
    +
    398 
    +        # Assert that nothing is cached locally anymore
    399 
    +        for element_name in all_elements:
    400 
    +            assert cli.get_element_state(project, element_name) != 'cached'
    401 
    +
    402 
    +        # Attempt bst build with --use-remotes set as none, this should lead to
    403 
    +        # a complete rebuild without pulling from either artifact remote cache
    404 
    +        result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
    405 
    +        result.assert_success()
    406 
    +        for element_name in all_elements:
    407 
    +            assert element_name not in result.get_pulled_elements()
    408 
    +
    409 
    +        # Delete local cache again
    410 
    +        artifacts = os.path.join(cli.directory, 'artifacts')
    411 
    +        shutil.rmtree(artifacts)
    412 
    +
    413 
    +        # Attempt bst build with --use-remotes set as user, as the shareuser is
    414 
    +        # passed in as user config and not via a project, assert project remote
    415 
    +        # was not attempted by it not being in the output
    416 
    +        result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
    417 
    +        result.assert_success()
    418 
    +        for element_name in all_elements:
    419 
    +            assert element_name in result.get_pulled_elements()
    420 
    +        assert shareproject.repo not in result.stderr
    421 
    +
    422 
    +        # Delete local cache again
    423 
    +        artifacts = os.path.join(cli.directory, 'artifacts')
    424 
    +        shutil.rmtree(artifacts)
    425 
    +
    426 
    +        # Attempt bst build with --use-remotes set as all, this time
    427 
    +        # assert that project remote is attempted and in the output
    428 
    +        result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
    429 
    +        result.assert_success()
    430 
    +        for element_name in all_elements:
    431 
    +            assert element_name in result.get_pulled_elements()
    432 
    +        assert shareproject.repo in result.stderr

  • tests/frontend/push.py
    ... ... @@ -409,3 +409,68 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
    409 409 
             assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"
    410 410 
             assert "INFO    Remote ({}) already has ".format(share.repo) in result.stderr
    411 411 
             assert "SKIPPED Push" in result.stderr
    412 
    +
    413 
    +
    414 
    +# Tests that:
    415 
    +#
    416 
    +#  * The bst main option --use-remotes limits remote action
    417 
    +#    as expected for push jobs
    418 
    +#
    419 
    +@pytest.mark.datafiles(DATA_DIR)
    420 
    +def test_useremotes_cli_options(cli, tmpdir, datafiles):
    421 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    422 
    +
    423 
    +    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
    424 
    +        create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
    425 
    +
    426 
    +        # Add shareproject repo url to project.conf
    427 
    +        with open(os.path.join(project, "project.conf"), "a") as projconf:
    428 
    +            projconf.write("artifacts:\n  url: {}\n  push: True".format(shareproject.repo))
    429 
    +
    430 
    +        # Configure shareuser remote in user conf
    431 
    +        cli.configure({
    432 
    +            'artifacts': {'url': shareuser.repo, 'push': True}
    433 
    +        })
    434 
    +
    435 
    +        # First build the target element with --use-remotes set as none.
    436 
    +        # This should lead to a complete build without pushing to either artifact
    437 
    +        # remote cache
    438 
    +        result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
    439 
    +        result.assert_success()
    440 
    +        assert not result.get_pushed_elements()
    441 
    +        assert cli.get_element_state(project, 'target.bst') == 'cached'
    442 
    +
    443 
    +        # Delete the artifacts from the local artifact cache
    444 
    +        all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
    445 
    +        for element_name in all_elements:
    446 
    +            cli.remove_artifact_from_cache(project, element_name)
    447 
    +
    448 
    +        # Assert that nothing is cached locally anymore
    449 
    +        for element_name in all_elements:
    450 
    +            assert cli.get_element_state(project, element_name) != 'cached'
    451 
    +
    452 
    +        # Attempt bst build with --use-remotes set as user, this should lead to
    453 
    +        # a complete rebuild, with artifacts pushed to the shareuser remote artifact cache
    454 
    +        # only. Assert project remote was not attempted by it not being in the output
    455 
    +        result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
    456 
    +        result.assert_success()
    457 
    +        for element_name in all_elements:
    458 
    +            assert element_name in result.get_pushed_elements()
    459 
    +        for element_name in all_elements:
    460 
    +            assert_shared(cli, shareuser, project, element_name)
    461 
    +        assert shareproject.repo not in result.stderr
    462 
    +
    463 
    +        # Delete the artifacts from the local artifact cache
    464 
    +        all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
    465 
    +        for element_name in all_elements:
    466 
    +            cli.remove_artifact_from_cache(project, element_name)
    467 
    +
    468 
    +        # Attempt bst build with --use-remotes set as all, this should lead to
    469 
    +        # a complete rebuild, with artifacts pushed to both the shareuser and
    470 
    +        # shareproject remote artifacts caches
    471 
    +        result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
    472 
    +        result.assert_success()
    473 
    +        for element_name in all_elements:
    474 
    +            assert element_name in result.get_pushed_elements()
    475 
    +        for element_name in all_elements:
    476 
    +            assert_shared(cli, shareproject, project, element_name)

  • tests/frontend/show.py
    ... ... @@ -36,6 +36,19 @@ def test_show(cli, datafiles, target, format, expected):
    36 36 
                                  .format(expected, result.output))
    37 37
    38 38
    39 
    +@pytest.mark.datafiles(os.path.join(
    40 
    +    os.path.dirname(os.path.realpath(__file__)),
    41 
    +    "invalid_element_path",
    42 
    +))
    43 
    +def test_show_invalid_element_path(cli, datafiles):
    44 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    45 
    +    result = cli.run(project=project, silent=True, args=[
    46 
    +        'show',
    47 
    +        "foo.bst"])
    48 
    +
    49 
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
    50 
    +
    51 
    +
    39 52 
     @pytest.mark.datafiles(DATA_DIR)
    40 53 
     @pytest.mark.parametrize("target,except_,expected", [
    41 54 
         ('target.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst', 'target.bst']),

  • tests/integration/project/elements/base/base-alpine.bst
    ... ... @@ -7,6 +7,11 @@ description: |
    7 7
    8 8 
     sources:
    9 9 
       - kind: tar
    10 
    -    url: alpine:integration-tests-base.v1.x86_64.tar.xz
    11 10 
         base-dir: ''
    12 
    -    ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
    11 
    +    (?):
    12 
    +    - arch == "x86_64":
    13 
    +        ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
    14 
    +        url: "alpine:integration-tests-base.v1.x86_64.tar.xz"
    15 
    +    - arch == "aarch64":
    16 
    +        ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a
    17 
    +        url: "alpine:integration-tests-base.v1.aarch64.tar.xz"

  • tests/integration/project/project.conf
    ... ... @@ -9,6 +9,12 @@ options:
    9 9 
         type: bool
    10 10 
         description: Whether to expect a linux platform
    11 11 
         default: True
    12 
    +  arch:
    13 
    +    type: arch
    14 
    +    description: Current architecture
    15 
    +    values:
    16 
    +      - x86_64
    17 
    +      - aarch64
    12 18 
     split-rules:
    13 19 
       test:
    14 20 
         - |

  • tests/testutils/site.py
    ... ... @@ -49,3 +49,5 @@ except ImportError:
    49 49 
         HAVE_ARPY = False
    50 50
    51 51 
     IS_LINUX = os.getenv('BST_FORCE_BACKEND', sys.platform).startswith('linux')
    52 
    +
    53 
    +_, _, _, _, MACHINE_ARCH = os.uname()

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]