GNOME.org
 

[Notes] [Git][BuildStream/buildstream][master] 6 commits: cli: Add support for auto-completing artifact ref names

Title: GitLab

richardmaw-codethink pushed to branch master at BuildStream / buildstream

Commits:

4 changed files:

Changes:

  • NEWS
    ... ... @@ -2,6 +2,8 @@
    2 2 
     buildstream 1.3.1
    3 3 
     =================
    4 4
    5 
    +  o Added `bst artifact log` subcommand for viewing build logs.
    6 
    +
    5 7 
       o BREAKING CHANGE: The bst source-bundle command has been removed. The
    6 8 
         functionality it provided has been replaced by the `--include-build-scripts`
    7 9 
         option of the `bst source-checkout` command. To produce a tarball containing

  • buildstream/_frontend/cli.py
    1 1 
     import os
    2 2 
     import sys
    3 
    +from contextlib import ExitStack
    4 
    +from fnmatch import fnmatch
    5 
    +from tempfile import TemporaryDirectory
    3 6
    4 7 
     import click
    5 8 
     from .. import _yaml
    ... ... @@ -107,6 +110,23 @@ def complete_target(args, incomplete):
    107 110 
         return complete_list
    108 111
    109 112
    113 
    +def complete_artifact(args, incomplete):
    114 
    +    from .._context import Context
    115 
    +    ctx = Context()
    116 
    +
    117 
    +    config = None
    118 
    +    for i, arg in enumerate(args):
    119 
    +        if arg in ('-c', '--config'):
    120 
    +            config = args[i + 1]
    121 
    +    ctx.load(config)
    122 
    +
    123 
    +    # element targets are valid artifact names
    124 
    +    complete_list = complete_target(args, incomplete)
    125 
    +    complete_list.extend(ref for ref in ctx.artifactcache.cas.list_refs() if ref.startswith(incomplete))
    126 
    +
    127 
    +    return complete_list
    128 
    +
    129 
    +
    110 130 
     def override_completions(cmd, cmd_param, args, incomplete):
    111 131 
         """
    112 132 
         :param cmd_param: command definition
    ... ... @@ -121,13 +141,15 @@ def override_completions(cmd, cmd_param, args, incomplete):
    121 141 
         # We can't easily extend click's data structures without
    122 142 
         # modifying click itself, so just do some weak special casing
    123 143 
         # right here and select which parameters we want to handle specially.
    124 
    -    if isinstance(cmd_param.type, click.Path) and \
    125 
    -       (cmd_param.name == 'elements' or
    126 
    -        cmd_param.name == 'element' or
    127 
    -        cmd_param.name == 'except_' or
    128 
    -        cmd_param.opts == ['--track'] or
    129 
    -        cmd_param.opts == ['--track-except']):
    130 
    -        return complete_target(args, incomplete)
    144 
    +    if isinstance(cmd_param.type, click.Path):
    145 
    +        if (cmd_param.name == 'elements' or
    146 
    +                cmd_param.name == 'element' or
    147 
    +                cmd_param.name == 'except_' or
    148 
    +                cmd_param.opts == ['--track'] or
    149 
    +                cmd_param.opts == ['--track-except']):
    150 
    +            return complete_target(args, incomplete)
    151 
    +        if cmd_param.name == 'artifacts':
    152 
    +            return complete_artifact(args, incomplete)
    131 153
    132 154 
         raise CompleteUnhandled()
    133 155
    ... ... @@ -915,3 +937,101 @@ def workspace_list(app):
    915 937
    916 938 
         with app.initialized():
    917 939 
             app.stream.workspace_list()
    940 
    +
    941 
    +
    942 
    +#############################################################
    943 
    +#                     Artifact Commands                     #
    944 
    +#############################################################
    945 
    +def _classify_artifacts(names, cas, project_directory):
    946 
    +    element_targets = []
    947 
    +    artifact_refs = []
    948 
    +    element_globs = []
    949 
    +    artifact_globs = []
    950 
    +
    951 
    +    for name in names:
    952 
    +        if name.endswith('.bst'):
    953 
    +            if any(c in "*?[" for c in name):
    954 
    +                element_globs.append(name)
    955 
    +            else:
    956 
    +                element_targets.append(name)
    957 
    +        else:
    958 
    +            if any(c in "*?[" for c in name):
    959 
    +                artifact_globs.append(name)
    960 
    +            else:
    961 
    +                artifact_refs.append(name)
    962 
    +
    963 
    +    if element_globs:
    964 
    +        for dirpath, _, filenames in os.walk(project_directory):
    965 
    +            for filename in filenames:
    966 
    +                element_path = os.path.join(dirpath, filename).lstrip(project_directory).lstrip('/')
    967 
    +                if any(fnmatch(element_path, glob) for glob in element_globs):
    968 
    +                    element_targets.append(element_path)
    969 
    +
    970 
    +    if artifact_globs:
    971 
    +        artifact_refs.extend(ref for ref in cas.list_refs()
    972 
    +                             if any(fnmatch(ref, glob) for glob in artifact_globs))
    973 
    +
    974 
    +    return element_targets, artifact_refs
    975 
    +
    976 
    +
    977 
    +@cli.group(short_help="Manipulate cached artifacts")
    978 
    +def artifact():
    979 
    +    """Manipulate cached artifacts"""
    980 
    +    pass
    981 
    +
    982 
    +
    983 
    +################################################################
    984 
    +#                     Artifact Log Command                     #
    985 
    +################################################################
    986 
    +@artifact.command(name='log', short_help="Show logs of an artifact")
    987 
    +@click.argument('artifacts', type=click.Path(), nargs=-1)
    988 
    +@click.pass_obj
    989 
    +def artifact_log(app, artifacts):
    990 
    +    """Show logs of all artifacts"""
    991 
    +    from .._exceptions import CASError
    992 
    +    from .._message import MessageType
    993 
    +    from .._pipeline import PipelineSelection
    994 
    +    from ..storage._casbaseddirectory import CasBasedDirectory
    995 
    +
    996 
    +    with ExitStack() as stack:
    997 
    +        stack.enter_context(app.initialized())
    998 
    +        cache = app.context.artifactcache
    999 
    +
    1000 
    +        elements, artifacts = _classify_artifacts(artifacts, cache.cas,
    1001 
    +                                                  app.project.directory)
    1002 
    +
    1003 
    +        vdirs = []
    1004 
    +        extractdirs = []
    1005 
    +        if artifacts:
    1006 
    +            for ref in artifacts:
    1007 
    +                try:
    1008 
    +                    cache_id = cache.cas.resolve_ref(ref, update_mtime=True)
    1009 
    +                    vdir = CasBasedDirectory(cache.cas, cache_id)
    1010 
    +                    vdirs.append(vdir)
    1011 
    +                except CASError as e:
    1012 
    +                    app._message(MessageType.WARN, "Artifact {} is not cached".format(ref), detail=str(e))
    1013 
    +                    continue
    1014 
    +        if elements:
    1015 
    +            elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
    1016 
    +            for element in elements:
    1017 
    +                if not element._cached():
    1018 
    +                    app._message(MessageType.WARN, "Element {} is not cached".format(element))
    1019 
    +                    continue
    1020 
    +                ref = cache.get_artifact_fullname(element, element._get_cache_key())
    1021 
    +                cache_id = cache.cas.resolve_ref(ref, update_mtime=True)
    1022 
    +                vdir = CasBasedDirectory(cache.cas, cache_id)
    1023 
    +                vdirs.append(vdir)
    1024 
    +
    1025 
    +        for vdir in vdirs:
    1026 
    +            # NOTE: If reading the logs feels unresponsive, here would be a good place to provide progress information.
    1027 
    +            logsdir = vdir.descend(["logs"])
    1028 
    +            td = stack.enter_context(TemporaryDirectory())
    1029 
    +            logsdir.export_files(td, can_link=True)
    1030 
    +            extractdirs.append(td)
    1031 
    +
    1032 
    +        for extractdir in extractdirs:
    1033 
    +            for log in (os.path.join(extractdir, log) for log in os.listdir(extractdir)):
    1034 
    +                # NOTE: Should click gain the ability to pass files to the pager this can be optimised.
    1035 
    +                with open(log) as f:
    1036 
    +                    data = f.read()
    1037 
    +                    click.echo_via_pager(data)

  • tests/completions/completions.py
    ... ... @@ -6,6 +6,7 @@ from tests.testutils import cli
    6 6 
     DATA_DIR = os.path.dirname(os.path.realpath(__file__))
    7 7
    8 8 
     MAIN_COMMANDS = [
    9 
    +    'artifact ',
    9 10 
         'build ',
    10 11 
         'checkout ',
    11 12 
         'fetch ',

  • tests/integration/artifact.py
    1 
    +#
    2 
    +#  Copyright (C) 2018 Codethink Limited
    3 
    +#  Copyright (C) 2018 Bloomberg Finance LP
    4 
    +#
    5 
    +#  This program is free software; you can redistribute it and/or
    6 
    +#  modify it under the terms of the GNU Lesser General Public
    7 
    +#  License as published by the Free Software Foundation; either
    8 
    +#  version 2 of the License, or (at your option) any later version.
    9 
    +#
    10 
    +#  This library is distributed in the hope that it will be useful,
    11 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    12 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    13 
    +#  Lesser General Public License for more details.
    14 
    +#
    15 
    +#  You should have received a copy of the GNU Lesser General Public
    16 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    17 
    +#
    18 
    +#  Authors: Richard Maw <richard maw codethink co uk>
    19 
    +#
    20 
    +
    21 
    +import os
    22 
    +import pytest
    23 
    +
    24 
    +from tests.testutils import cli_integration as cli
    25 
    +
    26 
    +
    27 
    +pytestmark = pytest.mark.integration
    28 
    +
    29 
    +
    30 
    +# Project directory
    31 
    +DATA_DIR = os.path.join(
    32 
    +    os.path.dirname(os.path.realpath(__file__)),
    33 
    +    "project",
    34 
    +)
    35 
    +
    36 
    +
    37 
    +@pytest.mark.integration
    38 
    +@pytest.mark.datafiles(DATA_DIR)
    39 
    +def test_artifact_log(cli, tmpdir, datafiles):
    40 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    41 
    +
    42 
    +    # Get the cache key of our test element
    43 
    +    result = cli.run(project=project, silent=True, args=[
    44 
    +        '--no-colors',
    45 
    +        'show', '--deps', 'none', '--format', '%{full-key}',
    46 
    +        'base.bst'
    47 
    +    ])
    48 
    +    key = result.output.strip()
    49 
    +
    50 
    +    # Ensure we have an artifact to read
    51 
    +    result = cli.run(project=project, args=['build', 'base.bst'])
    52 
    +    assert result.exit_code == 0
    53 
    +
    54 
    +    # Read the log via the element name
    55 
    +    result = cli.run(project=project, args=['artifact', 'log', 'base.bst'])
    56 
    +    assert result.exit_code == 0
    57 
    +    log = result.output
    58 
    +
    59 
    +    # Read the log via the key
    60 
    +    result = cli.run(project=project, args=['artifact', 'log', 'test/base/' + key])
    61 
    +    assert result.exit_code == 0
    62 
    +    assert log == result.output
    63 
    +
    64 
    +    # Read the log via glob
    65 
    +    result = cli.run(project=project, args=['artifact', 'log', 'test/base/*'])
    66 
    +    assert result.exit_code == 0
    67 
    +    # The artifact is cached under both a strong key and a weak key
    68 
    +    assert (log + log) == result.output

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]