[Notes] [Git][BuildStream/buildstream][issue_640-Build-All] 9 commits:

Phillip Smyth pushed to branch issue_640-Build-All at BuildStream / buildstream

Commits:

516e990e

by ctolentino8 at 2018-10-31T11:36:46Z

_frontend/app.py: Set correct element-path in interactive bst-init

b8a37a63

by Tristan Van Berkom at 2018-11-01T10:16:25Z

Merge branch 'chiaratolentino/bst-init-interactive-elementpath' into 'master'

_frontend/app.py: Set correct element-path in interactive bst-init

See merge request BuildStream/buildstream!912

b27b592a

by Benjamin Schubert at 2018-11-01T10:49:57Z

Remove dependency on pytest-runner

This includes a new command mimicking pytest-runner so that we
can drop this dependency

This was the only setup_requires dependency that we had and
will make like easier for people behind proxies

89ace5d7

by Benjamin Schubert at 2018-11-01T11:16:36Z

Merge branch 'bschubert/remove-pytest-runner' into 'master'

Remove dependency on pytest-runner

See merge request BuildStream/buildstream!886

4cfabce8

by Angelos Evripiotis at 2018-11-01T11:35:02Z

BREAK: manual.yaml: don't set any default env vars

Remove the 'MAKEFLAGS' and 'V' defaults.

Now that there is a specialised 'make' element, these make-specific
defaults don't make sense here. This element is meant to be a blank
slate for folks to build on.

Note that this is a breaking change, that will require some users to
make changes to their .bst files if they are expecting these environment
variable to be set.

_versions.py: BST_FORMAT_VERSION bumped to 18.

This fixes issue #718

48860aac

by Tristan Van Berkom at 2018-11-01T12:01:04Z

Merge branch 'tristan/empty-manual-element' into 'master'

BREAK: manual.yaml: don't set any default env vars

Closes #718

See merge request BuildStream/buildstream!916

d868b409

by Daniel Silverstone at 2018-11-01T13:40:24Z

_yaml.py: Implement `get()` for `ChainMap`

Since the core Python `ChainMap.get()` implements with:

    self[key] if key in self else default

The double-chain-lookup is expensive.  This simple change solves that
for our ChainMap derived structure.  As such it improves matters
for #466 somewhat.

Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>

7f79b9ce

by Tristan Van Berkom at 2018-11-01T14:25:57Z

Merge branch 'danielsilverstone-ct/improve-chainmap-get' into 'master'

_yaml.py: Implement `get()` for `ChainMap`

See merge request BuildStream/buildstream!919

4ebf4fd1
by knownexus at 2018-11-02T14:13:44Z
```
tmp-commit-message
```

Changes:

NEWS

@@ -2,6 +2,12 @@
  buildstream 1.3.1
  =================
 +  o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
 +    environment variables. There is already a 'make' element with the same
 +    variables. Note that this is a breaking change, it will require users to
 +    make changes to their .bst files if they are expecting these environment
 +    variables to be set.
++
    o Failed builds are included in the cache as well.
      `bst checkout` will provide anything in `%{install-root}`.
      A build including cached fails will cause any dependant elements

buildstream/_frontend/app.py

@@ -305,7 +305,6 @@ class App():
          directory = self._main_options['directory']
          directory = os.path.abspath(directory)
          project_path = os.path.join(directory, 'project.conf')
 -        elements_path = os.path.join(directory, element_path)
          try:
              # Abort if the project.conf already exists, unless `--force` was specified in `bst init`
@@ -335,6 +334,7 @@ class App():
                  raise AppError("Error creating project directory {}: {}".format(directory, e)) from e
              # Create the elements sub-directory if it doesnt exist
 +            elements_path = os.path.join(directory, element_path)
              try:
                  os.makedirs(elements_path, exist_ok=True)
              except IOError as e:

buildstream/_frontend/cli.py

@@ -305,10 +305,12 @@ def init(app, project_name, format_version, element_path, force):
                help="Allow tracking to cross junction boundaries")
  @click.option('--track-save', default=False, is_flag=True,
                help="Deprecated: This is ignored")
 +@click.option('--world', 'build_world', default=False, is_flag=True,
 +              help="Build every element in the project")
  @click.argument('elements', nargs=-1,
                  type=click.Path(readable=False))
  @click.pass_obj
 -def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
 +def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions, build_world):
      """Build elements in a pipeline"""
      if (track_except or track_cross_junctions) and not (track_ or track_all):
@@ -319,14 +321,13 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
      if track_save:
          click.echo("WARNING: --track-save is deprecated, saving is now unconditional", err=True)
 -    if track_all:
 -        track_ = elements
+-
      with app.initialized(session_name="Build"):
          app.stream.build(elements,
                           track_targets=track_,
 +                         track_all=track_all,
                           track_except=track_except,
                           track_cross_junctions=track_cross_junctions,
 +                         build_world=build_world,
                           build_all=all_)

buildstream/_stream.py

@@ -163,15 +163,29 @@ class Stream():
+     #
      def build(self, targets, *,
                track_targets=None,
 +              track_all=False,
                track_except=None,
                track_cross_junctions=False,
 +              build_world=False,
                build_all=False):
 -        if build_all:
++
 +        if build_world:
 +            for root, _, files in os.walk("elements"):
 +                for file in files:
 +                    if file.endswith(".bst"):
 +                        relDir = os.path.relpath(root, "elements")
 +                        relFile = os.path.join(relDir, file).strip("./")
 +                        targets = targets + (relFile,)
 +            selection = PipelineSelection.PLAN
 +        elif build_all:
              selection = PipelineSelection.ALL
          else:
              selection = PipelineSelection.PLAN
 +        if track_all:
 +            track_targets = targets
++
          elements, track_elements = \
              self._load(targets, track_targets,
                         selection=selection, track_selection=PipelineSelection.ALL,

buildstream/_versions.py

@@ -23,7 +23,7 @@
  # This version is bumped whenever enhancements are made
  # to the `project.conf` format or the core element format.
+ #
 -BST_FORMAT_VERSION = 17
 +BST_FORMAT_VERSION = 18
  # The base BuildStream artifact version

buildstream/_yaml.py

@@ -1049,6 +1049,12 @@ class ChainMap(collections.ChainMap):
          for key in clearable:
              del self[key]
 +    def get(self, key, default=None):
 +        try:
 +            return self[key]
 +        except KeyError:
 +            return default
++
  def node_chain_copy(source):
      copy = ChainMap({}, source)

buildstream/plugins/elements/manual.yaml

 -# No variables added for the manual element by default, set
 -# this if you plan to use make, and the sources cannot handle
 -# parallelization.
 -#
 -# variables:
 -#
 -#   notparallel: True
+-
  # Manual build element does not provide any default
  # build commands
  config:
@@ -28,14 +20,3 @@ config:
    strip-commands:
    - |
      %{strip-binaries}
+-
 -# Use max-jobs CPUs for building and enable verbosity
 -environment:
 -  MAKEFLAGS: -j%{max-jobs}
 -  V: 1
+-
 -# And dont consider MAKEFLAGS or V as something which may
 -# affect build output.
 -environment-nocache:
 -- MAKEFLAGS
 -- V

setup.py

@@ -39,6 +39,7 @@ if sys.version_info[0] != REQUIRED_PYTHON_MAJOR or sys.version_info[1] < REQUIRE
  try:
      from setuptools import setup, find_packages, Command
      from setuptools.command.easy_install import ScriptWriter
 +    from setuptools.command.test import test as TestCommand
  except ImportError:
      print("BuildStream requires setuptools in order to build. Install it using"
            " your package manager (usually python3-setuptools) or via pip (pip3"
@@ -219,9 +220,48 @@ class BuildGRPC(Command):
                          f.write(code)
 +#####################################################
 +#                   Pytest command                  #
 +#####################################################
 +class PyTest(TestCommand):
 +    """Defines a pytest command class to run tests from setup.py"""
++
 +    user_options = TestCommand.user_options + [
 +        ("addopts=", None, "Arguments to pass to pytest"),
 +        ('index-url=''build_grpc': BuildGRPC,
 +        'pytest': PyTest,
+     }
      cmdclass.update(versioneer.get_cmdclass())
      return cmdclass
@@ -305,6 +345,5 @@ setup(name='BuildStream',
            'grpcio >= 1.10',
        ],
        entry_points=bst_install_entry_points,
 -      setup_requires=['pytest-runner'],
        tests_require=dev_requires,
        zip_safe=False)

tests/frontend/init.py

@@ -3,6 +3,7 @@ import pytest
  from tests.testutils import cli
  from buildstream import _yaml
 +from buildstream._frontend.app import App
  from buildstream._exceptions import ErrorDomain, LoadErrorReason
  from buildstream._versions import BST_FORMAT_VERSION
@@ -98,3 +99,34 @@ def test_bad_element_path(cli, tmpdir, element_path):
          'init', '--project-name', 'foo', '--element-path', element_path
      ])
      result.assert_main_error(ErrorDomain.APP, 'invalid-element-path')
++
++
 +@pytest.mark.parametrize("element_path", [('foo'), ('foo/bar')])
 +def test_element_path_interactive(cli, tmp_path, monkeypatch, element_path):
 +    project = tmp_path
 +    project_conf_path = project.joinpath('project.conf')
++
 +    class DummyInteractiveApp(App):
 +        def __init__(self, *args, **kwargs):
 +            super().__init__(*args, **kwargs)
 +            self.interactive = True
++
 +        @classmethod
 +        def create(cls, *args, **kwargs):
 +            return DummyInteractiveApp(*args, **kwargs)
++
 +        def _init_project_interactive(self, *args, **kwargs):
 +            return ('project_name', '0', element_path)
++
 +    monkeypatch.setattr(App, 'create', DummyInteractiveApp.create)
++
 +    result = cli.run(project=str(project), args=['init'])
 +    result.assert_success()
++
 +    full_element_path = project.joinpath(element_path)
 +    assert full_element_path.exists()
++
 +    project_conf = _yaml.load(str(project_conf_path))
 +    assert project_conf['name'] == 'project_name'
 +    assert project_conf['format-version'] == '0'
 +    assert project_conf['element-path'] == element_path

[Notes] [Git][BuildStream/buildstream][issue_640-Build-All] 9 commits: _frontend/app.py: Set correct element-path in interactive bst-init

Phillip Smyth pushed to branch issue_640-Build-All at BuildStream / buildstream

Commits:

9 changed files:

Changes: