GNOME.org
 

[Notes] [Git][BuildStream/buildstream][tpollard/workspacebuildtree] 9 commits: tests/plugin/pipeline.py: Avoid using host user conf

Title: GitLab

Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream

Commits:

24 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -182,8 +182,8 @@ docs:
    182 182 
       stage: test
    183 183 
       variables:
    184 184 
         BST_EXT_URL: git+https://gitlab.com/BuildStream/bst-external.git
    185 
    -    BST_EXT_REF: 1d6ab71151b93c8cbc0a91a36ffe9270f3b835f1 # 0.5.1
    186 
    -    FD_SDK_REF: 88d7c22c2281b987faa02edd57df80d430eecf1f # 18.08.11-35-g88d7c22c
    185 
    +    BST_EXT_REF: 573843768f4d297f85dc3067465b3c7519a8dcc3 # 0.7.0
    186 
    +    FD_SDK_REF: 612f66e218445eee2b1a9d7dd27c9caba571612e # freedesktop-sdk-18.08.19-54-g612f66e2
    187 187 
       before_script:
    188 188 
       - |
    189 189 
         mkdir -p "${HOME}/.config"

  • NEWS
    ... ... @@ -63,6 +63,10 @@ buildstream 1.3.1
    63 63
    64 64 
       o Added new `bst source-checkout` command to checkout sources of an element.
    65 65
    66 
    +  o `bst workspace open` now supports the creation of multiple elements and
    67 
    +    allows the user to set a default location for their creation. This has meant
    68 
    +    that the new CLI is no longer backwards compatible with buildstream 1.2.
    69 
    +
    66 70
    67 71 
     =================
    68 72 
     buildstream 1.1.5

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -835,6 +835,20 @@ class ArtifactCache():
    835 835
    836 836 
             self.cas.link_ref(oldref, newref)
    837 837
    838 
    +    # checkout_artifact_subdir()
    839 
    +    #
    840 
    +    # Checkout given artifact subdir into provided directory
    841 
    +    #
    842 
    +    # Args:
    843 
    +    #     element (Element): The Element
    844 
    +    #     key (str): The cache key to use
    845 
    +    #     subdir (str): The subdir to checkout
    846 
    +    #     tmpdir (str): The dir to place the subdir content
    847 
    +    #
    848 
    +    def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
    849 
    +        ref = self.get_artifact_fullname(element, key)
    850 
    +        return self.cas.checkout_artifact_subdir(ref, subdir, tmpdir)
    851 
    +
    838 852 
         ################################################
    839 853 
         #               Local Private Methods          #
    840 854 
         ################################################

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -404,6 +404,21 @@ class CASCache():
    404 404
    405 405 
             return True
    406 406
    407 
    +    # checkout_artifact_subdir():
    408 
    +    #
    409 
    +    # Checkout given artifact subdir into provided directory
    410 
    +    #
    411 
    +    # Args:
    412 
    +    #     ref (str): The ref to check
    413 
    +    #     subdir (str): The subdir to checkout
    414 
    +    #     tmpdir (str): The dir to place the subdir content
    415 
    +    #
    416 
    +    def checkout_artifact_subdir(self, ref, subdir, tmpdir):
    417 
    +        tree = self.resolve_ref(ref)
    418 
    +        # This assumes that the subdir digest is present in the element tree
    419 
    +        subdirdigest = self._get_subdir(tree, subdir)
    420 
    +        self._checkout(tmpdir, subdirdigest)
    421 
    +
    407 422 
         # objpath():
    408 423 
         #
    409 424 
         # Return the path of an object based on its digest.

  • buildstream/_context.py
    ... ... @@ -59,6 +59,9 @@ class Context():
    59 59 
             # The directory where build sandboxes will be created
    60 60 
             self.builddir = None
    61 61
    62 
    +        # Default root location for workspaces
    63 
    +        self.workspacedir = None
    64 
    +
    62 65 
             # The local binary artifact cache directory
    63 66 
             self.artifactdir = None
    64 67
    ... ... @@ -122,6 +125,9 @@ class Context():
    122 125 
             # a hard reset of a workspace, potentially losing changes.
    123 126 
             self.prompt_workspace_reset_hard = None
    124 127
    128 
    +        # Whether to not include artifact buildtrees in workspaces if available
    129 
    +        self.workspace_buildtrees = True
    130 
    +
    125 131 
             # Whether elements must be rebuilt when their dependencies have changed
    126 132 
             self._strict_build_plan = None
    127 133
    ... ... @@ -177,10 +183,10 @@ class Context():
    177 183 
             _yaml.node_validate(defaults, [
    178 184 
                 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    179 185 
                 'scheduler', 'artifacts', 'logging', 'projects',
    180 
    -            'cache', 'prompt'
    186 
    +            'cache', 'prompt', 'workspacedir', 'workspace-buildtrees'
    181 187 
             ])
    182 188
    183 
    -        for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
    189 
    +        for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
    184 190 
                 # Allow the ~ tilde expansion and any environment variables in
    185 191 
                 # path specification in the config files.
    186 192 
                 #
    ... ... @@ -205,6 +211,9 @@ class Context():
    205 211 
             # Load pull build trees configuration
    206 212 
             self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
    207 213
    214 
    +        # Load workspace buildtrees configuration
    215 
    +        self.workspace_buildtrees = _yaml.node_get(defaults, bool, 'workspace-buildtrees', default_value='True')
    216 
    +
    208 217 
             # Load logging config
    209 218 
             logging = _yaml.node_get(defaults, Mapping, 'logging')
    210 219 
             _yaml.node_validate(logging, [

  • buildstream/_frontend/cli.py
    ... ... @@ -705,33 +705,34 @@ def workspace():
    705 705 
     ##################################################################
    706 706 
     @workspace.command(name='open', short_help="Open a new workspace")
    707 707 
     @click.option('--no-checkout', default=False, is_flag=True,
    708 
    -              help="Do not checkout the source, only link to the given directory")
    708 
    +              help="Do not checkout the source or cached buildtree, only link to the given directory")
    709 709 
     @click.option('--force', '-f', default=False, is_flag=True,
    710 
    -              help="Overwrite files existing in checkout directory")
    710 
    +              help="The workspace will be created even if the directory in which it will be created is not empty " +
    711 
    +              "or if a workspace for that element already exists")
    711 712 
     @click.option('--track', 'track_', default=False, is_flag=True,
    712 713 
                   help="Track and fetch new source references before checking out the workspace")
    713 
    -@click.argument('element',
    714 
    -                type=click.Path(readable=False))
    715 
    -@click.argument('directory', type=click.Path(file_okay=False))
    714 
    +@click.option('--directory', type=click.Path(file_okay=False), default=None,
    715 
    +              help="Only for use when a single Element is given: Set the directory to use to create the workspace")
    716 
    +@click.argument('elements', nargs=-1, type=click.Path(readable=False), required=True)
    717 
    +@click.option('--no-cache', default=False, is_flag=True,
    718 
    +              help="Do not checkout the cached buildtree")
    716 719 
     @click.pass_obj
    717 
    -def workspace_open(app, no_checkout, force, track_, element, directory):
    718 
    -    """Open a workspace for manual source modification"""
    720 
    +def workspace_open(app, no_checkout, force, track_, directory, elements, no_cache):
    719 721
    720 
    -    if os.path.exists(directory):
    721 
    -
    722 
    -        if not os.path.isdir(directory):
    723 
    -            click.echo("Checkout directory is not a directory: {}".format(directory), err=True)
    724 
    -            sys.exit(-1)
    722 
    +    """Open a workspace for manual source modification, the elements buildtree
    723 
    +    will be provided if available in the local artifact cache.
    724 
    +    """
    725 725
    726 
    -        if not (no_checkout or force) and os.listdir(directory):
    727 
    -            click.echo("Checkout directory is not empty: {}".format(directory), err=True)
    728 
    -            sys.exit(-1)
    726 
    +    if not no_cache and not no_checkout:
    727 
    +        click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
    729 728
    730 729 
         with app.initialized():
    731 
    -        app.stream.workspace_open(element, directory,
    730 
    +        app.stream.workspace_open(elements,
    732 731 
                                       no_checkout=no_checkout,
    733 732 
                                       track_first=track_,
    734 
    -                                  force=force)
    733 
    +                                  force=force,
    734 
    +                                  custom_dir=directory,
    735 
    +                                  no_cache=no_cache)
    735 736
    736 737
    737 738 
     ##################################################################

  • buildstream/_stream.py
    ... ... @@ -464,44 +464,37 @@ class Stream():
    464 464 
         # Open a project workspace
    465 465 
         #
    466 466 
         # Args:
    467 
    -    #    target (str): The target element to open the workspace for
    468 
    -    #    directory (str): The directory to stage the source in
    467 
    +    #    targets (list): List of target elements to open workspaces for
    469 468 
         #    no_checkout (bool): Whether to skip checking out the source
    470 469 
         #    track_first (bool): Whether to track and fetch first
    471 470 
         #    force (bool): Whether to ignore contents in an existing directory
    471 
    +    #    custom_dir (str): Custom location to create a workspace or false to use default location.
    472 
    +    #    no_cache (bool): Whether to not include the cached buildtree
    472 473 
         #
    473 
    -    def workspace_open(self, target, directory, *,
    474 
    +    def workspace_open(self, targets, *,
    474 475 
                            no_checkout,
    475 476 
                            track_first,
    476 
    -                       force):
    477 
    +                       force,
    478 
    +                       custom_dir,
    479 
    +                       no_cache):
    480 
    +
    481 
    +        # This function is a little funny but it is trying to be as atomic as possible.
    482 
    +
    483 
    +        # Set no_cache if the global user conf workspacebuildtrees is false
    484 
    +        if not self._context.workspace_buildtrees:
    485 
    +            no_cache = True
    477 486
    478 487 
             if track_first:
    479 
    -            track_targets = (target,)
    488 
    +            track_targets = targets
    480 489 
             else:
    481 490 
                 track_targets = ()
    482 491
    483 
    -        elements, track_elements = self._load((target,), track_targets,
    492 
    +        elements, track_elements = self._load(targets, track_targets,
    484 493 
                                                   selection=PipelineSelection.REDIRECT,
    485 494 
                                                   track_selection=PipelineSelection.REDIRECT)
    486 
    -        target = elements[0]
    487 
    -        directory = os.path.abspath(directory)
    488 
    -
    489 
    -        if not list(target.sources()):
    490 
    -            build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
    491 
    -            if not build_depends:
    492 
    -                raise StreamError("The given element has no sources")
    493 
    -            detail = "Try opening a workspace on one of its dependencies instead:\n"
    494 
    -            detail += "  \n".join(build_depends)
    495 
    -            raise StreamError("The given element has no sources", detail=detail)
    496 495
    497 496 
             workspaces = self._context.get_workspaces()
    498 497
    499 
    -        # Check for workspace config
    500 
    -        workspace = workspaces.get_workspace(target._get_full_name())
    501 
    -        if workspace and not force:
    502 
    -            raise StreamError("Workspace '{}' is already defined at: {}"
    503 
    -                              .format(target.name, workspace.get_absolute_path()))
    504 
    -
    505 498 
             # If we're going to checkout, we need at least a fetch,
    506 499 
             # if we were asked to track first, we're going to fetch anyway.
    507 500 
             #
    ... ... @@ -511,29 +504,107 @@ class Stream():
    511 504 
                     track_elements = elements
    512 505 
                 self._fetch(elements, track_elements=track_elements)
    513 506
    514 
    -        if not no_checkout and target._get_consistency() != Consistency.CACHED:
    515 
    -            raise StreamError("Could not stage uncached source. " +
    516 
    -                              "Use `--track` to track and " +
    517 
    -                              "fetch the latest version of the " +
    518 
    -                              "source.")
    519 
    -
    520 
    -        if workspace:
    521 
    -            workspaces.delete_workspace(target._get_full_name())
    522 
    -            workspaces.save_config()
    523 
    -            shutil.rmtree(directory)
    524 
    -        try:
    525 
    -            os.makedirs(directory, exist_ok=True)
    526 
    -        except OSError as e:
    527 
    -            raise StreamError("Failed to create workspace directory: {}".format(e)) from e
    528 
    -
    529 
    -        workspaces.create_workspace(target._get_full_name(), directory)
    507 
    +        expanded_directories = []
    508 
    +        #  To try to be more atomic, loop through the elements and raise any errors we can early
    509 
    +        for target in elements:
    510 
    +
    511 
    +            if not list(target.sources()):
    512 
    +                build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
    513 
    +                if not build_depends:
    514 
    +                    raise StreamError("The element {}  has no sources".format(target.name))
    515 
    +                detail = "Try opening a workspace on one of its dependencies instead:\n"
    516 
    +                detail += "  \n".join(build_depends)
    517 
    +                raise StreamError("The element {} has no sources".format(target.name), detail=detail)
    518 
    +
    519 
    +            # Check for workspace config
    520 
    +            workspace = workspaces.get_workspace(target._get_full_name())
    521 
    +            if workspace and not force:
    522 
    +                raise StreamError("Element '{}' already has workspace defined at: {}"
    523 
    +                                  .format(target.name, workspace.get_absolute_path()))
    524 
    +
    525 
    +            if not no_checkout and target._get_consistency() != Consistency.CACHED:
    526 
    +                raise StreamError("Could not stage uncached source. For {} ".format(target.name) +
    527 
    +                                  "Use `--track` to track and " +
    528 
    +                                  "fetch the latest version of the " +
    529 
    +                                  "source.")
    530 
    +
    531 
    +            if not custom_dir:
    532 
    +                directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name))
    533 
    +                if directory[-4:] == '.bst':
    534 
    +                    directory = directory[:-4]
    535 
    +                expanded_directories.append(directory)
    536 
    +
    537 
    +        if custom_dir:
    538 
    +            if len(elements) != 1:
    539 
    +                raise StreamError("Exactly one element can be given if --directory is used",
    540 
    +                                  reason='directory-with-multiple-elements')
    541 
    +            expanded_directories = [custom_dir, ]
    542 
    +        else:
    543 
    +            # If this fails it is a bug in whatever calls this, usually cli.py and so can not be tested for via the
    544 
    +            # run bst test mechanism.
    545 
    +            assert len(elements) == len(expanded_directories)
    546 
    +
    547 
    +        for target, directory in zip(elements, expanded_directories):
    548 
    +            if os.path.exists(directory):
    549 
    +                if not os.path.isdir(directory):
    550 
    +                    raise StreamError("For element '{}', Directory path is not a directory: {}"
    551 
    +                                      .format(target.name, directory), reason='bad-directory')
    552 
    +
    553 
    +                if not (no_checkout or force) and os.listdir(directory):
    554 
    +                    raise StreamError("For element '{}', Directory path is not empty: {}"
    555 
    +                                      .format(target.name, directory), reason='bad-directory')
    556 
    +
    557 
    +        # So far this function has tried to catch as many issues as possible with out making any changes
    558 
    +        # Now it does the bits that can not be made atomic.
    559 
    +        targetGenerator = zip(elements, expanded_directories)
    560 
    +        for target, directory in targetGenerator:
    561 
    +            self._message(MessageType.INFO, "Creating workspace for element {}"
    562 
    +                          .format(target.name))
    563 
    +
    564 
    +            # Check if given target has a buildtree artifact cached locally
    565 
    +            buildtree = None
    566 
    +            if target._cached():
    567 
    +                buildtree = self._artifacts.contains_subdir_artifact(target, target._get_cache_key(), 'buildtree')
    568 
    +
    569 
    +            # If we're running in the default state, make the user aware of buildtree usage
    570 
    +            if not no_cache and not no_checkout:
    571 
    +                if buildtree:
    572 
    +                    self._message(MessageType.INFO, "{} buildtree artifact is available,"
    573 
    +                                  " workspace will be opened with it".format(target.name))
    574 
    +                else:
    575 
    +                    self._message(MessageType.WARN, "{} buildtree artifact not available,"
    576 
    +                                  " workspace will be opened with source checkout".format(target.name))
    530 577
    531 
    -        if not no_checkout:
    532 
    -            with target.timed_activity("Staging sources to {}".format(directory)):
    533 
    -                target._open_workspace()
    578 
    +            workspace = workspaces.get_workspace(target._get_full_name())
    579 
    +            if workspace:
    580 
    +                workspaces.delete_workspace(target._get_full_name())
    581 
    +                workspaces.save_config()
    582 
    +                shutil.rmtree(directory)
    583 
    +            try:
    584 
    +                os.makedirs(directory, exist_ok=True)
    585 
    +            except OSError as e:
    586 
    +                todo_elements = " ".join([str(target.name) for target, directory_dict in targetGenerator])
    587 
    +                if todo_elements:
    588 
    +                    # This output should make creating the remaining workspaces as easy as possible.
    589 
    +                    todo_elements = "\nDid not try to create workspaces for " + todo_elements
    590 
    +                raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
    591 
    +
    592 
    +            # Handle opening workspace with buildtree included
    593 
    +            if (buildtree and not no_cache) and not no_checkout:
    594 
    +                workspaces.create_workspace(target._get_full_name(), directory, cached_build=buildtree)
    595 
    +                with target.timed_activity("Staging buildtree to {}".format(directory)):
    596 
    +                    target._open_workspace(buildtree=buildtree)
    597 
    +            else:
    598 
    +                workspaces.create_workspace(target._get_full_name(), directory)
    599 
    +                if (not buildtree or no_cache) and not no_checkout:
    600 
    +                    with target.timed_activity("Staging sources to {}".format(directory)):
    601 
    +                        target._open_workspace()
    534 602
    535 
    -        workspaces.save_config()
    536 
    -        self._message(MessageType.INFO, "Saved workspace configuration")
    603 
    +            # Saving the workspace once it is set up means that if the next workspace fails to be created before
    604 
    +            # the configuration gets saved. The successfully created workspace still gets saved.
    605 
    +            workspaces.save_config()
    606 
    +            self._message(MessageType.INFO, "Created a workspace for element: {}"
    607 
    +                          .format(target._get_full_name()))
    537 608
    538 609 
         # workspace_close
    539 610 
         #
    ... ... @@ -614,10 +685,24 @@ class Stream():
    614 685 
                                           .format(workspace_path, e)) from e
    615 686
    616 687 
                 workspaces.delete_workspace(element._get_full_name())
    617 
    -            workspaces.create_workspace(element._get_full_name(), workspace_path)
    618 688
    619 
    -            with element.timed_activity("Staging sources to {}".format(workspace_path)):
    620 
    -                element._open_workspace()
    689 
    +            # Create the workspace, ensuring the original optional cached build state is preserved if
    690 
    +            # possible.
    691 
    +            buildtree = False
    692 
    +            if workspace.cached_build and element._cached():
    693 
    +                if self._artifacts.contains_subdir_artifact(element, element._get_cache_key(), 'buildtree'):
    694 
    +                    buildtree = True
    695 
    +
    696 
    +            # Warn the user if the workspace cannot be opened with the original cached build state
    697 
    +            if workspace.cached_build and not buildtree:
    698 
    +                self._message(MessageType.WARN, "{} original buildtree artifact not available,"
    699 
    +                              " workspace will be opened with source checkout".format(element.name))
    700 
    +
    701 
    +            workspaces.create_workspace(element._get_full_name(), workspace_path,
    702 
    +                                        cached_build=buildtree)
    703 
    +
    704 
    +            with element.timed_activity("Staging to {}".format(workspace_path)):
    705 
    +                element._open_workspace(buildtree=buildtree)
    621 706
    622 707 
                 self._message(MessageType.INFO,
    623 708 
                               "Reset workspace for {} at: {}".format(element.name,

  • buildstream/_workspaces.py
    ... ... @@ -24,7 +24,7 @@ from . import _yaml
    24 24 
     from ._exceptions import LoadError, LoadErrorReason
    25 25
    26 26
    27 
    -BST_WORKSPACE_FORMAT_VERSION = 3
    27 
    +BST_WORKSPACE_FORMAT_VERSION = 4
    28 28
    29 29
    30 30 
     # Workspace()
    ... ... @@ -43,9 +43,11 @@ BST_WORKSPACE_FORMAT_VERSION = 3
    43 43 
     #    running_files (dict): A dict mapping dependency elements to files
    44 44 
     #                          changed between failed builds. Should be
    45 45 
     #                          made obsolete with failed build artifacts.
    46 
    +#    cached_build (bool): If the workspace is staging the cached build artifact
    46 47 
     #
    47 48 
     class Workspace():
    48 
    -    def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
    49 
    +    def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False,
    50 
    +                 running_files=None, cached_build=False):
    49 51 
             self.prepared = prepared
    50 52 
             self.last_successful = last_successful
    51 53 
             self._path = path
    ... ... @@ -53,6 +55,7 @@ class Workspace():
    53 55
    54 56 
             self._toplevel_project = toplevel_project
    55 57 
             self._key = None
    58 
    +        self.cached_build = cached_build
    56 59
    57 60 
         # to_dict()
    58 61 
         #
    ... ... @@ -65,7 +68,8 @@ class Workspace():
    65 68 
             ret = {
    66 69 
                 'prepared': self.prepared,
    67 70 
                 'path': self._path,
    68 
    -            'running_files': self.running_files
    71 
    +            'running_files': self.running_files,
    72 
    +            'cached_build': self.cached_build
    69 73 
             }
    70 74 
             if self.last_successful is not None:
    71 75 
                 ret["last_successful"] = self.last_successful
    ... ... @@ -224,12 +228,13 @@ class Workspaces():
    224 228 
         # Args:
    225 229 
         #    element_name (str) - The element name to create a workspace for
    226 230 
         #    path (str) - The path in which the workspace should be kept
    231 
    +    #    cached_build (bool) - If the workspace is staging the cached build artifact
    227 232 
         #
    228 
    -    def create_workspace(self, element_name, path):
    233 
    +    def create_workspace(self, element_name, path, cached_build=False):
    229 234 
             if path.startswith(self._toplevel_project.directory):
    230 235 
                 path = os.path.relpath(path, self._toplevel_project.directory)
    231 236
    232 
    -        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
    237 
    +        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path, cached_build=cached_build)
    233 238
    234 239 
             return self._workspaces[element_name]
    235 240
    ... ... @@ -396,6 +401,7 @@ class Workspaces():
    396 401 
                 'path': _yaml.node_get(node, str, 'path'),
    397 402 
                 'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
    398 403 
                 'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
    404 
    +            'cached_build': _yaml.node_get(node, bool, 'cached_build', default_value=False)
    399 405 
             }
    400 406 
             return Workspace.from_dict(self._toplevel_project, dictionary)
    401 407

  • buildstream/data/userconfig.yaml
    ... ... @@ -22,6 +22,9 @@ artifactdir: ${XDG_CACHE_HOME}/buildstream/artifacts
    22 22 
     # Location to store build logs
    23 23 
     logdir: ${XDG_CACHE_HOME}/buildstream/logs
    24 24
    25 
    +# Default root location for workspaces, blank for no default set.
    26 
    +workspacedir: .
    27 
    +
    25 28 
     #
    26 29 
     #    Cache
    27 30 
     #

  • buildstream/element.py
    ... ... @@ -1880,7 +1880,10 @@ class Element(Plugin):
    1880 1880 
         # This requires that a workspace already be created in
    1881 1881 
         # the workspaces metadata first.
    1882 1882 
         #
    1883 
    -    def _open_workspace(self):
    1883 
    +    # Args:
    1884 
    +    #    buildtree (bool): Whether to open workspace with artifact buildtree
    1885 
    +    #
    1886 
    +    def _open_workspace(self, buildtree=False):
    1884 1887 
             context = self._get_context()
    1885 1888 
             workspace = self._get_workspace()
    1886 1889 
             assert workspace is not None
    ... ... @@ -1893,11 +1896,19 @@ class Element(Plugin):
    1893 1896 
             # files in the target directory actually works without any
    1894 1897 
             # additional support from Source implementations.
    1895 1898 
             #
    1899 
    +
    1896 1900 
             os.makedirs(context.builddir, exist_ok=True)
    1897 1901 
             with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
    1898 1902 
                                 .format(self.normal_name)) as temp:
    1899 
    -            for source in self.sources():
    1900 
    -                source._init_workspace(temp)
    1903 
    +
    1904 
    +            # Checkout cached buildtree, augment with source plugin if applicable
    1905 
    +            if buildtree:
    1906 
    +                self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', temp)
    1907 
    +                for source in self.sources():
    1908 
    +                    source._init_cached_build_workspace(temp)
    1909 
    +            else:
    1910 
    +                for source in self.sources():
    1911 
    +                    source._init_workspace(temp)
    1901 1912
    1902 1913 
                 # Now hardlink the files into the workspace target.
    1903 1914 
                 utils.link_files(temp, workspace.get_absolute_path())

  • buildstream/plugins/sources/git.py
    ... ... @@ -268,6 +268,35 @@ class GitMirror(SourceFetcher):
    268 268 
             if track:
    269 269 
                 self.assert_ref_in_track(fullpath, track)
    270 270
    271 
    +    def init_cached_build_workspace(self, directory, track=None):
    272 
    +        fullpath = os.path.join(directory, self.path)
    273 
    +        url = self.source.translate_url(self.url)
    274 
    +
    275 
    +        self.source.call([self.source.host_git, 'init', fullpath],
    276 
    +                         fail="Failed to init git in directory: {}".format(fullpath),
    277 
    +                         fail_temporarily=True,
    278 
    +                         cwd=fullpath)
    279 
    +
    280 
    +        self.source.call([self.source.host_git, 'fetch', self.mirror],
    281 
    +                         fail='Failed to fetch from local mirror "{}"'.format(self.mirror),
    282 
    +                         cwd=fullpath)
    283 
    +
    284 
    +        self.source.call([self.source.host_git, 'remote', 'add', 'origin', url],
    285 
    +                         fail='Failed to add remote origin "{}"'.format(url),
    286 
    +                         cwd=fullpath)
    287 
    +
    288 
    +        self.source.call([self.source.host_git, 'update-ref', '--no-deref', 'HEAD', self.ref],
    289 
    +                         fail='Failed update HEAD to ref "{}"'.format(self.ref),
    290 
    +                         cwd=fullpath)
    291 
    +
    292 
    +        self.source.call([self.source.host_git, 'read-tree', 'HEAD'],
    293 
    +                         fail='Failed to read HEAD into index',
    294 
    +                         cwd=fullpath)
    295 
    +
    296 
    +        # Check that the user specified ref exists in the track if provided & not already tracked
    297 
    +        if track:
    298 
    +            self.assert_ref_in_track(fullpath, track)
    299 
    +
    271 300 
         # List the submodules (path/url tuples) present at the given ref of this repo
    272 301 
         def submodule_list(self):
    273 302 
             modules = "{}:{}".format(self.ref, GIT_MODULES)
    ... ... @@ -480,6 +509,14 @@ class GitSource(Source):
    480 509 
                 for mirror in self.submodules:
    481 510 
                     mirror.init_workspace(directory)
    482 511
    512 
    +    def init_cached_build_workspace(self, directory):
    513 
    +        self.refresh_submodules()
    514 
    +
    515 
    +        with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
    516 
    +            self.mirror.init_cached_build_workspace(directory, track=(self.tracking if not self.tracked else None))
    517 
    +            for mirror in self.submodules:
    518 
    +                mirror.init_cached_build_workspace(directory)
    519 
    +
    483 520 
         def stage(self, directory):
    484 521
    485 522 
             # Need to refresh submodule list here again, because

  • buildstream/source.py
    ... ... @@ -459,6 +459,24 @@ class Source(Plugin):
    459 459 
             """
    460 460 
             self.stage(directory)
    461 461
    462 
    +    def init_cached_build_workspace(self, directory):
    463 
    +        """Initialises a new cached build workspace
    464 
    +
    465 
    +        Args:
    466 
    +           directory (str): Path of the workspace to init
    467 
    +
    468 
    +        Raises:
    469 
    +           :class:`.SourceError`
    470 
    +
    471 
    +        Implementors overriding this method should assume that *directory*
    472 
    +        already exists.
    473 
    +
    474 
    +        Implementors should raise :class:`.SourceError` when encountering
    475 
    +        some system error.
    476 
    +        """
    477 
    +        # Allow a non implementation
    478 
    +        return None
    479 
    +
    462 480 
         def get_source_fetchers(self):
    463 481 
             """Get the objects that are used for fetching
    464 482
    ... ... @@ -677,6 +695,12 @@ class Source(Plugin):
    677 695
    678 696 
             self.init_workspace(directory)
    679 697
    698 
    +    # Wrapper for init_cached_build_workspace()
    699 
    +    def _init_cached_build_workspace(self, directory):
    700 
    +        directory = self.__ensure_directory(directory)
    701 
    +
    702 
    +        self.init_cached_build_workspace(directory)
    703 
    +
    680 704 
         # _get_unique_key():
    681 705 
         #
    682 706 
         # Wrapper for get_unique_key() api

  • doc/sessions/developing.run
    ... ... @@ -7,7 +7,7 @@ commands:
    7 7 
     # Capture workspace open output
    8 8 
     - directory: ../examples/developing/
    9 9 
       output: ../source/sessions/developing-workspace-open.html
    10 
    -  command: workspace open hello.bst workspace_hello
    10 
    +  command: workspace open --directory workspace_hello hello.bst
    11 11
    12 12 
     # Catpure output from workspace list
    13 13 
     - directory: ../examples/developing/
    ... ... @@ -37,7 +37,7 @@ commands:
    37 37 
     # Reopen workspace
    38 38 
     - directory: ../examples/developing/
    39 39 
       output: ../source/sessions/developing-reopen-workspace.html
    40 
    -  command: workspace open --no-checkout hello.bst workspace_hello
    40 
    +  command: workspace open --no-checkout --directory workspace_hello hello.bst
    41 41
    42 42 
     # Reset workspace
    43 43 
     - directory: ../examples/developing/

  • doc/sessions/junctions.run
    ... ... @@ -13,7 +13,7 @@ commands:
    13 13 
     # Open a crossJunction workspace:
    14 14 
     - directory: ../examples/junctions
    15 15 
       output: ../source/sessions/junctions-workspace-open.html
    16 
    -  command: workspace open hello-junction.bst:hello.bst workspace_hello
    16 
    +  command: workspace open --directory workspace_hello hello-junction.bst:hello.bst
    17 17
    18 18 
     # Remove the workspace
    19 19 
     - directory: ../examples/junctions

  • doc/source/developing/workspaces.rst
    ... ... @@ -24,9 +24,32 @@ Suppose we now want to alter the functionality of the *hello* command. We can
    24 24 
     make changes to the source code of Buildstream elements by making use of
    25 25 
     BuildStream's workspace command.
    26 26
    27 
    +Utilising cached buildtrees
    28 
    +---------------------------
    29 
    + When a BuildStream build element artifact is created and cached, a snapshot of
    30 
    + the build directory after the build commands have completed is included in the
    31 
    + artifact. This `build tree` can be considered an intermediary state of element,
    32 
    + where the source is present along with any output created during the build
    33 
    + execution.
    34 
    +
    35 
    + By default when opening a workspace, bst will attempt to stage the build tree
    36 
    + into the workspace if it's available in the local cache. If the respective
    37 
    + build tree is not present in the cache (element not cached, partially cached or
    38 
    + is a non build element) then the source will be staged as is. The default
    39 
    + behaviour to attempt to use the build tree can be overriden with specific bst
    40 
    + workspace open option of `--no-cache`, or via setting user configuration option
    41 
    + `workspacebuildtrees: False`
    42 
    +
    27 43
    28 44 
     Opening a workspace
    29 45 
     -------------------
    46 
    +.. note::
    47 
    +
    48 
    +    This example presumes you built the hello.bst during
    49 
    +    :ref:`running commands <tutorial_running_commands>`
    50 
    +    if not, please start by building it.
    51 
    +
    52 
    +
    30 53 
     First we need to open a workspace, we can do this by running
    31 54
    32 55 
     .. raw:: html
    ... ... @@ -88,6 +111,15 @@ Alternatively, if we wish to discard the changes we can use
    88 111
    89 112 
     This resets the workspace to its original state.
    90 113
    114 
    +.. note::
    115 
    +
    116 
    +    bst reset will attempt to open the workspace in
    117 
    +    the condition in which it was originally staged,
    118 
    +    i.e with or without consuming the element build tree.
    119 
    +    If it was originally staged with a cached build tree
    120 
    +    and there's no longer one available, the source will
    121 
    +    be staged as is.
    122 
    +
    91 123 
     To discard the workspace completely we can do:
    92 124
    93 125 
     .. raw:: html

  • tests/examples/developing.py
    ... ... @@ -59,7 +59,7 @@ def test_open_workspace(cli, tmpdir, datafiles):
    59 59 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    60 60 
         workspace_dir = os.path.join(str(tmpdir), "workspace_hello")
    61 61
    62 
    -    result = cli.run(project=project, args=['workspace', 'open', '-f', 'hello.bst', workspace_dir])
    62 
    +    result = cli.run(project=project, args=['workspace', 'open', '-f', '--directory', workspace_dir, 'hello.bst', ])
    63 63 
         result.assert_success()
    64 64
    65 65 
         result = cli.run(project=project, args=['workspace', 'list'])
    ... ... @@ -78,7 +78,7 @@ def test_make_change_in_workspace(cli, tmpdir, datafiles):
    78 78 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    79 79 
         workspace_dir = os.path.join(str(tmpdir), "workspace_hello")
    80 80
    81 
    -    result = cli.run(project=project, args=['workspace', 'open', '-f', 'hello.bst', workspace_dir])
    81 
    +    result = cli.run(project=project, args=['workspace', 'open', '-f', '--directory', workspace_dir, 'hello.bst'])
    82 82 
         result.assert_success()
    83 83
    84 84 
         result = cli.run(project=project, args=['workspace', 'list'])

  • tests/examples/junctions.py
    ... ... @@ -48,7 +48,7 @@ def test_open_cross_junction_workspace(cli, tmpdir, datafiles):
    48 48 
         workspace_dir = os.path.join(str(tmpdir), "workspace_hello_junction")
    49 49
    50 50 
         result = cli.run(project=project,
    51 
    -                     args=['workspace', 'open', 'hello-junction.bst:hello.bst', workspace_dir])
    51 
    +                     args=['workspace', 'open', '--directory', workspace_dir, 'hello-junction.bst:hello.bst'])
    52 52 
         result.assert_success()
    53 53
    54 54 
         result = cli.run(project=project,

  • tests/frontend/buildcheckout.py
    ... ... @@ -509,7 +509,7 @@ def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles):
    509 509
    510 510 
         # Now open a workspace on the junction
    511 511 
         #
    512 
    -    result = cli.run(project=project, args=['workspace', 'open', 'junction.bst', workspace])
    512 
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, 'junction.bst'])
    513 513 
         result.assert_success()
    514 514 
         filename = os.path.join(workspace, 'files', 'etc-files', 'etc', 'animal.conf')
    515 515

  • tests/frontend/cross_junction_workspace.py
    ... ... @@ -47,7 +47,7 @@ def open_cross_junction(cli, tmpdir):
    47 47 
         workspace = tmpdir.join("workspace")
    48 48
    49 49 
         element = 'sub.bst:data.bst'
    50 
    -    args = ['workspace', 'open', element, str(workspace)]
    50 
    +    args = ['workspace', 'open', '--directory', str(workspace), element]
    51 51 
         result = cli.run(project=project, args=args)
    52 52 
         result.assert_success()
    53 53

  • tests/frontend/workspace.py
    ... ... @@ -21,9 +21,11 @@
    21 21 
     #           Phillip Smyth <phillip smyth codethink co uk>
    22 22 
     #           Jonathan Maw <jonathan maw codethink co uk>
    23 23 
     #           Richard Maw <richard maw codethink co uk>
    24 
    +#           William Salmon <will salmon codethink co uk>
    24 25 
     #
    25 26
    26 27 
     import os
    28 
    +import stat
    27 29 
     import pytest
    28 30 
     import shutil
    29 31 
     import subprocess
    ... ... @@ -43,65 +45,123 @@ DATA_DIR = os.path.join(
    43 45 
     )
    44 46
    45 47
    46 
    -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
    47 
    -                   project_path=None, element_attrs=None):
    48 
    -    if not workspace_dir:
    49 
    -        workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
    50 
    -    if not project_path:
    51 
    -        project_path = os.path.join(datafiles.dirname, datafiles.basename)
    52 
    -    else:
    53 
    -        shutil.copytree(os.path.join(datafiles.dirname, datafiles.basename), project_path)
    54 
    -    bin_files_path = os.path.join(project_path, 'files', 'bin-files')
    55 
    -    element_path = os.path.join(project_path, 'elements')
    56 
    -    element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
    57 
    -
    58 
    -    # Create our repo object of the given source type with
    59 
    -    # the bin files, and then collect the initial ref.
    60 
    -    #
    61 
    -    repo = create_repo(kind, str(tmpdir))
    62 
    -    ref = repo.create(bin_files_path)
    63 
    -    if track:
    64 
    -        ref = None
    65 
    -
    66 
    -    # Write out our test target
    67 
    -    element = {
    68 
    -        'kind': 'import',
    69 
    -        'sources': [
    70 
    -            repo.source_config(ref=ref)
    71 
    -        ]
    72 
    -    }
    73 
    -    if element_attrs:
    74 
    -        element = {**element, **element_attrs}
    75 
    -    _yaml.dump(element,
    76 
    -               os.path.join(element_path,
    77 
    -                            element_name))
    78 
    -
    79 
    -    # Assert that there is no reference, a track & fetch is needed
    80 
    -    state = cli.get_element_state(project_path, element_name)
    81 
    -    if track:
    82 
    -        assert state == 'no reference'
    83 
    -    else:
    84 
    -        assert state == 'fetch needed'
    85 
    -
    86 
    -    # Now open the workspace, this should have the effect of automatically
    87 
    -    # tracking & fetching the source from the repo.
    88 
    -    args = ['workspace', 'open']
    89 
    -    if track:
    90 
    -        args.append('--track')
    91 
    -    args.extend([element_name, workspace_dir])
    92 
    -    result = cli.run(project=project_path, args=args)
    93 
    -
    94 
    -    result.assert_success()
    48 
    +class WorkspaceCreater():
    49 
    +    def __init__(self, cli, tmpdir, datafiles, project_path=None):
    50 
    +        self.cli = cli
    51 
    +        self.tmpdir = tmpdir
    52 
    +        self.datafiles = datafiles
    53 
    +
    54 
    +        if not project_path:
    55 
    +            project_path = os.path.join(datafiles.dirname, datafiles.basename)
    56 
    +        else:
    57 
    +            shutil.copytree(os.path.join(datafiles.dirname, datafiles.basename), project_path)
    58 
    +
    59 
    +        self.project_path = project_path
    60 
    +        self.bin_files_path = os.path.join(project_path, 'files', 'bin-files')
    61 
    +
    62 
    +        self.workspace_cmd = os.path.join(self.project_path, 'workspace_cmd')
    63 
    +
    64 
    +    def create_workspace_element(self, kind, track, suffix='', workspace_dir=None,
    65 
    +                                 element_attrs=None):
    66 
    +        element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
    67 
    +        element_path = os.path.join(self.project_path, 'elements')
    68 
    +        if not workspace_dir:
    69 
    +            workspace_dir = os.path.join(self.workspace_cmd, element_name)
    70 
    +            if workspace_dir[-4:] == '.bst':
    71 
    +                workspace_dir = workspace_dir[:-4]
    72 
    +
    73 
    +        # Create our repo object of the given source type with
    74 
    +        # the bin files, and then collect the initial ref.
    75 
    +        repo = create_repo(kind, str(self.tmpdir))
    76 
    +        ref = repo.create(self.bin_files_path)
    77 
    +        if track:
    78 
    +            ref = None
    79 
    +
    80 
    +        # Write out our test target
    81 
    +        element = {
    82 
    +            'kind': 'import',
    83 
    +            'sources': [
    84 
    +                repo.source_config(ref=ref)
    85 
    +            ]
    86 
    +        }
    87 
    +        if element_attrs:
    88 
    +            element = {**element, **element_attrs}
    89 
    +        _yaml.dump(element,
    90 
    +                   os.path.join(element_path,
    91 
    +                                element_name))
    92 
    +        return element_name, element_path, workspace_dir
    93 
    +
    94 
    +    def create_workspace_elements(self, kinds, track, suffixes=None, workspace_dir_usr=None,
    95 
    +                                  element_attrs=None):
    96 
    +
    97 
    +        element_tuples = []
    98 
    +
    99 
    +        if suffixes is None:
    100 
    +            suffixes = ['', ] * len(kinds)
    101 
    +        else:
    102 
    +            if len(suffixes) != len(kinds):
    103 
    +                raise "terable error"
    104 
    +
    105 
    +        for suffix, kind in zip(suffixes, kinds):
    106 
    +            element_name, element_path, workspace_dir = \
    107 
    +                self.create_workspace_element(kind, track, suffix, workspace_dir_usr,
    108 
    +                                              element_attrs)
    109 
    +
    110 
    +            # Assert that there is no reference, a track & fetch is needed
    111 
    +            state = self.cli.get_element_state(self.project_path, element_name)
    112 
    +            if track:
    113 
    +                assert state == 'no reference'
    114 
    +            else:
    115 
    +                assert state == 'fetch needed'
    116 
    +            element_tuples.append((element_name, workspace_dir))
    117 
    +
    118 
    +        return element_tuples
    119 
    +
    120 
    +    def open_workspaces(self, kinds, track, suffixes=None, workspace_dir=None,
    121 
    +                        element_attrs=None, no_cache=False):
    122 
    +
    123 
    +        element_tuples = self.create_workspace_elements(kinds, track, suffixes, workspace_dir,
    124 
    +                                                        element_attrs)
    125 
    +        os.makedirs(self.workspace_cmd, exist_ok=True)
    126 
    +
    127 
    +        # Now open the workspace, this should have the effect of automatically
    128 
    +        # tracking & fetching the source from the repo.
    129 
    +        args = ['workspace', 'open']
    130 
    +        if track:
    131 
    +            args.append('--track')
    132 
    +        if no_cache:
    133 
    +            args.append('--no-cache')
    134 
    +        if workspace_dir is not None:
    135 
    +            assert len(element_tuples) == 1, "test logic error"
    136 
    +            _, workspace_dir = element_tuples[0]
    137 
    +            args.extend(['--directory', workspace_dir])
    138 
    +        print("element_tuples", element_tuples)
    139 
    +        args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
    140 
    +        print("args", args)
    141 
    +        result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
    142 
    +
    143 
    +        result.assert_success()
    144 
    +
    145 
    +        for element_name, workspace_dir in element_tuples:
    146 
    +            # Assert that we are now buildable because the source is
    147 
    +            # now cached.
    148 
    +            assert self.cli.get_element_state(self.project_path, element_name) == 'buildable'
    149 
    +
    150 
    +            # Check that the executable hello file is found in the workspace
    151 
    +            filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
    152 
    +            assert os.path.exists(filename)
    153 
    +
    154 
    +        return element_tuples, result
    95 155
    96 
    -    # Assert that we are now buildable because the source is
    97 
    -    # now cached.
    98 
    -    assert cli.get_element_state(project_path, element_name) == 'buildable'
    99 156
    100 
    -    # Check that the executable hello file is found in the workspace
    101 
    -    filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
    102 
    -    assert os.path.exists(filename)
    103 
    -
    104 
    -    return (element_name, project_path, workspace_dir)
    157 
    +def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
    158 
    +                   project_path=None, element_attrs=None, no_cache=False):
    159 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles, project_path)
    160 
    +    workspaces, _ = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
    161 
    +                                                     element_attrs, no_cache)
    162 
    +    assert len(workspaces) == 1
    163 
    +    element_name, workspace = workspaces[0]
    164 
    +    return element_name, workspace_object.project_path, workspace
    105 165
    106 166
    107 167 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -128,6 +188,128 @@ def test_open_bzr_customize(cli, tmpdir, datafiles):
    128 188 
         assert(expected_output_str in str(output))
    129 189
    130 190
    191 
    +@pytest.mark.datafiles(DATA_DIR)
    192 
    +def test_open_multi(cli, tmpdir, datafiles):
    193 
    +
    194 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    195 
    +    workspaces, _ = workspace_object.open_workspaces(repo_kinds, False)
    196 
    +
    197 
    +    for (elname, workspace), kind in zip(workspaces, repo_kinds):
    198 
    +        assert kind in elname
    199 
    +        workspace_lsdir = os.listdir(workspace)
    200 
    +        if kind == 'git':
    201 
    +            assert('.git' in workspace_lsdir)
    202 
    +        elif kind == 'bzr':
    203 
    +            assert('.bzr' in workspace_lsdir)
    204 
    +        else:
    205 
    +            assert not ('.git' in workspace_lsdir)
    206 
    +            assert not ('.bzr' in workspace_lsdir)
    207 
    +
    208 
    +
    209 
    +@pytest.mark.datafiles(DATA_DIR)
    210 
    +def test_open_multi_unwritable(cli, tmpdir, datafiles):
    211 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    212 
    +
    213 
    +    element_tuples = workspace_object.create_workspace_elements(repo_kinds, False, repo_kinds)
    214 
    +    os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
    215 
    +
    216 
    +    # Now open the workspace, this should have the effect of automatically
    217 
    +    # tracking & fetching the source from the repo.
    218 
    +    args = ['workspace', 'open']
    219 
    +    args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
    220 
    +    cli.configure({'workspacedir': workspace_object.workspace_cmd})
    221 
    +
    222 
    +    cwdstat = os.stat(workspace_object.workspace_cmd)
    223 
    +    try:
    224 
    +        os.chmod(workspace_object.workspace_cmd, cwdstat.st_mode - stat.S_IWRITE)
    225 
    +        result = workspace_object.cli.run(project=workspace_object.project_path, args=args)
    226 
    +    finally:
    227 
    +        # Using this finally to make sure we always put thing back how they should be.
    228 
    +        os.chmod(workspace_object.workspace_cmd, cwdstat.st_mode)
    229 
    +
    230 
    +    result.assert_main_error(ErrorDomain.STREAM, None)
    231 
    +    # Normally we avoid checking stderr in favour of using the mechine readable result.assert_main_error
    232 
    +    # But Tristan was very keen that the names of the elements left needing workspaces were present in the out put
    233 
    +    assert (" ".join([element_name for element_name, workspace_dir_suffix in element_tuples[1:]]) in result.stderr)
    234 
    +
    235 
    +
    236 
    +@pytest.mark.datafiles(DATA_DIR)
    237 
    +def test_open_multi_with_directory(cli, tmpdir, datafiles):
    238 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    239 
    +
    240 
    +    element_tuples = workspace_object.create_workspace_elements(repo_kinds, False, repo_kinds)
    241 
    +    os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
    242 
    +
    243 
    +    # Now open the workspace, this should have the effect of automatically
    244 
    +    # tracking & fetching the source from the repo.
    245 
    +    args = ['workspace', 'open']
    246 
    +    args.extend(['--directory', 'any/dir/should/fail'])
    247 
    +
    248 
    +    args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
    249 
    +    result = workspace_object.cli.run(cwd=workspace_object.workspace_cmd, project=workspace_object.project_path,
    250 
    +                                      args=args)
    251 
    +
    252 
    +    result.assert_main_error(ErrorDomain.STREAM, 'directory-with-multiple-elements')
    253 
    +
    254 
    +
    255 
    +@pytest.mark.datafiles(DATA_DIR)
    256 
    +def test_open_defaultlocation(cli, tmpdir, datafiles):
    257 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    258 
    +
    259 
    +    ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], False, ['git'])
    260 
    +    os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
    261 
    +
    262 
    +    # Now open the workspace, this should have the effect of automatically
    263 
    +    # tracking & fetching the source from the repo.
    264 
    +    args = ['workspace', 'open']
    265 
    +    args.append(element_name)
    266 
    +
    267 
    +    # In the other tests we set the cmd to workspace_object.workspace_cmd with the optional
    268 
    +    # argument, cwd for the workspace_object.cli.run function. But hear we set the default
    269 
    +    # workspace location to workspace_object.workspace_cmd and run the cli.run function with
    270 
    +    # no cwd option so that it runs in the project directory.
    271 
    +    cli.configure({'workspacedir': workspace_object.workspace_cmd})
    272 
    +    result = workspace_object.cli.run(project=workspace_object.project_path,
    273 
    +                                      args=args)
    274 
    +
    275 
    +    result.assert_success()
    276 
    +
    277 
    +    assert cli.get_element_state(workspace_object.project_path, element_name) == 'buildable'
    278 
    +
    279 
    +    # Check that the executable hello file is found in the workspace
    280 
    +    # even though the cli.run function was not run with cwd = workspace_object.workspace_cmd
    281 
    +    # the workspace should be created in there as we used the 'workspacedir' configuration
    282 
    +    # option.
    283 
    +    filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
    284 
    +    assert os.path.exists(filename)
    285 
    +
    286 
    +
    287 
    +@pytest.mark.datafiles(DATA_DIR)
    288 
    +def test_open_defaultlocation_exists(cli, tmpdir, datafiles):
    289 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    290 
    +
    291 
    +    ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], False, ['git'])
    292 
    +    os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
    293 
    +
    294 
    +    with open(workspace_dir, 'w') as fl:
    295 
    +        fl.write('foo')
    296 
    +
    297 
    +    # Now open the workspace, this should have the effect of automatically
    298 
    +    # tracking & fetching the source from the repo.
    299 
    +    args = ['workspace', 'open']
    300 
    +    args.append(element_name)
    301 
    +
    302 
    +    # In the other tests we set the cmd to workspace_object.workspace_cmd with the optional
    303 
    +    # argument, cwd for the workspace_object.cli.run function. But hear we set the default
    304 
    +    # workspace location to workspace_object.workspace_cmd and run the cli.run function with
    305 
    +    # no cwd option so that it runs in the project directory.
    306 
    +    cli.configure({'workspacedir': workspace_object.workspace_cmd})
    307 
    +    result = workspace_object.cli.run(project=workspace_object.project_path,
    308 
    +                                      args=args)
    309 
    +
    310 
    +    result.assert_main_error(ErrorDomain.STREAM, 'bad-directory')
    311 
    +
    312 
    +
    131 313 
     @pytest.mark.datafiles(DATA_DIR)
    132 314 
     @pytest.mark.parametrize("kind", repo_kinds)
    133 315 
     def test_open_track(cli, tmpdir, datafiles, kind):
    ... ... @@ -150,7 +332,7 @@ def test_open_force(cli, tmpdir, datafiles, kind):
    150 332
    151 333 
         # Now open the workspace again with --force, this should happily succeed
    152 334 
         result = cli.run(project=project, args=[
    153 
    -        'workspace', 'open', '--force', element_name, workspace
    335 
    +        'workspace', 'open', '--force', '--directory', workspace, element_name
    154 336 
         ])
    155 337 
         result.assert_success()
    156 338
    ... ... @@ -165,7 +347,7 @@ def test_open_force_open(cli, tmpdir, datafiles, kind):
    165 347
    166 348 
         # Now open the workspace again with --force, this should happily succeed
    167 349 
         result = cli.run(project=project, args=[
    168 
    -        'workspace', 'open', '--force', element_name, workspace
    350 
    +        'workspace', 'open', '--force', '--directory', workspace, element_name
    169 351 
         ])
    170 352 
         result.assert_success()
    171 353
    ... ... @@ -196,7 +378,7 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
    196 378
    197 379 
         # Now open the workspace again with --force, this should happily succeed
    198 380 
         result = cli.run(project=project, args=[
    199 
    -        'workspace', 'open', '--force', element_name2, workspace
    381 
    +        'workspace', 'open', '--force', '--directory', workspace, element_name2
    200 382 
         ])
    201 383
    202 384 
         # Assert that the file in workspace 1 has been replaced
    ... ... @@ -504,7 +686,7 @@ def test_buildable_no_ref(cli, tmpdir, datafiles):
    504 686 
         # Now open the workspace. We don't need to checkout the source though.
    505 687 
         workspace = os.path.join(str(tmpdir), 'workspace-no-ref')
    506 688 
         os.makedirs(workspace)
    507 
    -    args = ['workspace', 'open', '--no-checkout', element_name, workspace]
    689 
    +    args = ['workspace', 'open', '--no-checkout', '--directory', workspace, element_name]
    508 690 
         result = cli.run(project=project, args=args)
    509 691 
         result.assert_success()
    510 692
    ... ... @@ -637,7 +819,9 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    637 819 
                 "alpha.bst": {
    638 820 
                     "prepared": False,
    639 821 
                     "path": "/workspaces/bravo",
    640 
    -                "running_files": {}
    822 
    +                "running_files": {},
    823 
    +                "cached_build": False
    824 
    +
    641 825 
                 }
    642 826 
             }
    643 827 
         }),
    ... ... @@ -652,7 +836,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    652 836 
                 "alpha.bst": {
    653 837 
                     "prepared": False,
    654 838 
                     "path": "/workspaces/bravo",
    655 
    -                "running_files": {}
    839 
    +                "running_files": {},
    840 
    +                "cached_build": False
    656 841 
                 }
    657 842 
             }
    658 843 
         }),
    ... ... @@ -670,7 +855,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    670 855 
                 "alpha.bst": {
    671 856 
                     "prepared": False,
    672 857 
                     "path": "/workspaces/bravo",
    673 
    -                "running_files": {}
    858 
    +                "running_files": {},
    859 
    +                "cached_build": False
    674 860 
                 }
    675 861 
             }
    676 862 
         }),
    ... ... @@ -695,7 +881,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    695 881 
                     "last_successful": "some_key",
    696 882 
                     "running_files": {
    697 883 
                         "beta.bst": ["some_file"]
    698 
    -                }
    884 
    +                },
    885 
    +                "cached_build": False
    699 886 
                 }
    700 887 
             }
    701 888 
         }),
    ... ... @@ -715,7 +902,30 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    715 902 
                 "alpha.bst": {
    716 903 
                     "prepared": True,
    717 904 
                     "path": "/workspaces/bravo",
    718 
    -                "running_files": {}
    905 
    +                "running_files": {},
    906 
    +                "cached_build": False
    907 
    +            }
    908 
    +        }
    909 
    +    }),
    910 
    +    # Test loading version 4
    911 
    +    ({
    912 
    +        "format-version": 4,
    913 
    +        "workspaces": {
    914 
    +            "alpha.bst": {
    915 
    +                "prepared": False,
    916 
    +                "path": "/workspaces/bravo",
    917 
    +                "running_files": {},
    918 
    +                "cached_build": True
    919 
    +            }
    920 
    +        }
    921 
    +    }, {
    922 
    +        "format-version": BST_WORKSPACE_FORMAT_VERSION,
    923 
    +        "workspaces": {
    924 
    +            "alpha.bst": {
    925 
    +                "prepared": False,
    926 
    +                "path": "/workspaces/bravo",
    927 
    +                "running_files": {},
    928 
    +                "cached_build": True
    719 929 
                 }
    720 930 
             }
    721 931 
         })
    ... ... @@ -766,7 +976,7 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte
    766 976 
                                 element_name))
    767 977
    768 978 
         # Make a change to the workspaces file
    769 
    -    result = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    979 
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    770 980 
         result.assert_success()
    771 981 
         result = cli.run(project=project, args=['workspace', 'close', '--remove-dir', element_name])
    772 982 
         result.assert_success()
    ... ... @@ -876,3 +1086,80 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
    876 1086 
             result = cli.run(project=project, args=["build", element_name])
    877 1087 
             assert "BUG" not in result.stderr
    878 1088 
             assert cli.get_element_state(project, element_name) != "cached"
    1089 
    +
    1090 
    +
    1091 
    +@pytest.mark.datafiles(DATA_DIR)
    1092 
    +def test_nocache_open_messages(cli, tmpdir, datafiles):
    1093 
    +
    1094 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    1095 
    +    _, result = workspace_object.open_workspaces(('git', ), False)
    1096 
    +
    1097 
    +    # cli default WARN for source dropback possibility when no-cache flag is not passed
    1098 
    +    assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" in result.output
    1099 
    +
    1100 
    +    # cli WARN for source dropback happening when no-cache flag not given, but buildtree not available
    1101 
    +    assert "workspace will be opened with source checkout" in result.stderr
    1102 
    +
    1103 
    +    # cli default WARN for source dropback possibilty not given when no-cache flag is passed
    1104 
    +    tmpdir = os.path.join(str(tmpdir), "2")
    1105 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    1106 
    +    _, result = workspace_object.open_workspaces(('git', ), False, suffixes='1', no_cache=True)
    1107 
    +
    1108 
    +    assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" not in result.output
    1109 
    +
    1110 
    +
    1111 
    +@pytest.mark.datafiles(DATA_DIR)
    1112 
    +def test_nocache_reset_messages(cli, tmpdir, datafiles):
    1113 
    +
    1114 
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    1115 
    +    workspaces, result = workspace_object.open_workspaces(('git', ), False)
    1116 
    +    element_name, workspace = workspaces[0]
    1117 
    +    project = workspace_object.project_path
    1118 
    +
    1119 
    +    # Modify workspace, without building so the artifact is not cached
    1120 
    +    shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
    1121 
    +    os.makedirs(os.path.join(workspace, 'etc'))
    1122 
    +    with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
    1123 
    +        f.write("PONY='pink'")
    1124 
    +
    1125 
    +    # Now reset the open workspace, this should have the
    1126 
    +    # effect of reverting our changes to the original source, as it
    1127 
    +    # was not originally opened with a cached buildtree and as such
    1128 
    +    # should not notify the user
    1129 
    +    result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
    1130 
    +        'workspace', 'reset', element_name
    1131 
    +    ])
    1132 
    +    result.assert_success()
    1133 
    +    assert "original buildtree artifact not available" not in result.output
    1134 
    +    assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
    1135 
    +    assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
    1136 
    +
    1137 
    +    # Close the workspace
    1138 
    +    result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
    1139 
    +        'workspace', 'close', '--remove-dir', element_name
    1140 
    +    ])
    1141 
    +    result.assert_success()
    1142 
    +
    1143 
    +    # Build the workspace so we have a cached buildtree artifact for the element
    1144 
    +    assert cli.get_element_state(project, element_name) == 'buildable'
    1145 
    +    result = cli.run(project=project, args=['build', element_name])
    1146 
    +    result.assert_success()
    1147 
    +
    1148 
    +    # Opening the workspace after a build should lead to the cached buildtree being
    1149 
    +    # staged by default
    1150 
    +    result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
    1151 
    +        'workspace', 'open', element_name
    1152 
    +    ])
    1153 
    +    result.assert_success()
    1154 
    +
    1155 
    +    result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
    1156 
    +        'workspace', 'list'
    1157 
    +    ])
    1158 
    +    result.assert_success()
    1159 
    +    # Now reset the workspace and ensure that a warning is not given about the artifact
    1160 
    +    # buildtree not being available
    1161 
    +    result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
    1162 
    +        'workspace', 'reset', element_name
    1163 
    +    ])
    1164 
    +    result.assert_success()
    1165 
    +    assert "original buildtree artifact not available" not in result.output

  • tests/integration/shell.py
    ... ... @@ -290,7 +290,7 @@ def test_workspace_visible(cli, tmpdir, datafiles):
    290 290
    291 291 
         # Open a workspace on our build failing element
    292 292 
         #
    293 
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    293 
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    294 294 
         assert res.exit_code == 0
    295 295
    296 296 
         # Ensure the dependencies of our build failing element are built

  • tests/integration/workspace.py
    ... ... @@ -24,7 +24,7 @@ def test_workspace_mount(cli, tmpdir, datafiles):
    24 24 
         workspace = os.path.join(cli.directory, 'workspace')
    25 25 
         element_name = 'workspace/workspace-mount.bst'
    26 26
    27 
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    27 
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    28 28 
         assert res.exit_code == 0
    29 29
    30 30 
         res = cli.run(project=project, args=['build', element_name])
    ... ... @@ -41,7 +41,7 @@ def test_workspace_commanddir(cli, tmpdir, datafiles):
    41 41 
         workspace = os.path.join(cli.directory, 'workspace')
    42 42 
         element_name = 'workspace/workspace-commanddir.bst'
    43 43
    44 
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    44 
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    45 45 
         assert res.exit_code == 0
    46 46
    47 47 
         res = cli.run(project=project, args=['build', element_name])
    ... ... @@ -78,7 +78,7 @@ def test_workspace_updated_dependency(cli, tmpdir, datafiles):
    78 78 
         _yaml.dump(dependency, os.path.join(element_path, dep_name))
    79 79
    80 80 
         # First open the workspace
    81 
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    81 
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    82 82 
         assert res.exit_code == 0
    83 83
    84 84 
         # We build the workspaced element, so that we have an artifact
    ... ... @@ -134,7 +134,7 @@ def test_workspace_update_dependency_failed(cli, tmpdir, datafiles):
    134 134 
         _yaml.dump(dependency, os.path.join(element_path, dep_name))
    135 135
    136 136 
         # First open the workspace
    137 
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    137 
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    138 138 
         assert res.exit_code == 0
    139 139
    140 140 
         # We build the workspaced element, so that we have an artifact
    ... ... @@ -210,7 +210,7 @@ def test_updated_dependency_nested(cli, tmpdir, datafiles):
    210 210 
         _yaml.dump(dependency, os.path.join(element_path, dep_name))
    211 211
    212 212 
         # First open the workspace
    213 
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    213 
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    214 214 
         assert res.exit_code == 0
    215 215
    216 216 
         # We build the workspaced element, so that we have an artifact
    ... ... @@ -264,7 +264,7 @@ def test_incremental_configure_commands_run_only_once(cli, tmpdir, datafiles):
    264 264 
         _yaml.dump(element, os.path.join(element_path, element_name))
    265 265
    266 266 
         # We open a workspace on the above element
    267 
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    267 
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    268 268 
         res.assert_success()
    269 269
    270 270 
         # Then we build, and check whether the configure step succeeded

  • tests/plugins/filter.py
    ... ... @@ -108,19 +108,28 @@ def test_filter_forbid_also_rdep(datafiles, cli):
    108 108 
     def test_filter_workspace_open(datafiles, cli, tmpdir):
    109 109 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    110 110 
         workspace_dir = os.path.join(tmpdir.dirname, tmpdir.basename, "workspace")
    111 
    -    result = cli.run(project=project, args=['workspace', 'open', 'deps-permitted.bst', workspace_dir])
    111 
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'deps-permitted.bst'])
    112 112 
         result.assert_success()
    113 113 
         assert os.path.exists(os.path.join(workspace_dir, "foo"))
    114 114 
         assert os.path.exists(os.path.join(workspace_dir, "bar"))
    115 115 
         assert os.path.exists(os.path.join(workspace_dir, "baz"))
    116 116
    117 117
    118 
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
    119 
    +def test_filter_workspace_open_multi(datafiles, cli, tmpdir):
    120 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    121 
    +    result = cli.run(cwd=project, project=project, args=['workspace', 'open', 'deps-permitted.bst',
    122 
    +                                                         'output-orphans.bst'])
    123 
    +    result.assert_success()
    124 
    +    assert os.path.exists(os.path.join(project, "input"))
    125 
    +
    126 
    +
    118 127 
     @pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
    119 128 
     def test_filter_workspace_build(datafiles, cli, tmpdir):
    120 129 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    121 130 
         tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
    122 131 
         workspace_dir = os.path.join(tempdir, "workspace")
    123 
    -    result = cli.run(project=project, args=['workspace', 'open', 'output-orphans.bst', workspace_dir])
    132 
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
    124 133 
         result.assert_success()
    125 134 
         src = os.path.join(workspace_dir, "foo")
    126 135 
         dst = os.path.join(workspace_dir, "quux")
    ... ... @@ -138,7 +147,7 @@ def test_filter_workspace_close(datafiles, cli, tmpdir):
    138 147 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    139 148 
         tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
    140 149 
         workspace_dir = os.path.join(tempdir, "workspace")
    141 
    -    result = cli.run(project=project, args=['workspace', 'open', 'output-orphans.bst', workspace_dir])
    150 
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
    142 151 
         result.assert_success()
    143 152 
         src = os.path.join(workspace_dir, "foo")
    144 153 
         dst = os.path.join(workspace_dir, "quux")
    ... ... @@ -158,7 +167,7 @@ def test_filter_workspace_reset(datafiles, cli, tmpdir):
    158 167 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    159 168 
         tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
    160 169 
         workspace_dir = os.path.join(tempdir, "workspace")
    161 
    -    result = cli.run(project=project, args=['workspace', 'open', 'output-orphans.bst', workspace_dir])
    170 
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
    162 171 
         result.assert_success()
    163 172 
         src = os.path.join(workspace_dir, "foo")
    164 173 
         dst = os.path.join(workspace_dir, "quux")

  • tests/plugins/pipeline.py
    ... ... @@ -14,7 +14,7 @@ DATA_DIR = os.path.join(
    14 14
    15 15 
     def create_pipeline(tmpdir, basedir, target):
    16 16 
         context = Context()
    17 
    -    context.load()
    17 
    +    context.load(config=os.devnull)
    18 18 
         context.deploydir = os.path.join(str(tmpdir), 'deploy')
    19 19 
         context.artifactdir = os.path.join(str(tmpdir), 'artifact')
    20 20 
         project = Project(basedir, context)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]