Merge branch 'master' into simbuerg/fix-544-buildah-cannot-find-image

.clog.toml

@@ -1,4 +1,4 @@
 [clog]
 repository = "https://github.com/PolyJIT/benchbuild"
-changelog = "docs/CHANGELOG.md"
+changelog = "docs/source/CHANGELOG.md"
 from-latest-tag = true

.coveragerc

@@ -7,6 +7,7 @@ branch = True
 parallel = True
 
 [paths]
+data_file = ${BB_COVERAGE_PATH-default .}/.coverage
 source =
   benchbuild/
   */site-packages/benchbuild/

.github/workflows/ci.yml

@@ -10,13 +10,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -53,7 +53,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10"]
+        db_support: [true, false]
 
     steps:
     - uses: actions/checkout@v2
@@ -76,21 +77,29 @@ jobs:
 
     - name: Run integration tests
       env:
-        BB_CONTAINER_ROOT: '/tmp'
-        BB_CONTAINER_RUNROOT: '/tmp'
+        BB_DB_ENABLED: ${{ matrix.db_support }}
+        BB_DB_CONNECT_STRING: "sqlite://"
+        BB_COVERAGE_COLLECT: true
+        BB_COVERAGE_PATH: ${{ github.workspace }}
+        BB_COVERAGE_CONFIG: ${{ github.workspace }}/.coveragerc
+        BB_CONTAINER_ROOT: ${{ runner.temp }}
+        BB_CONTAINER_RUNROOT: ${{ runner.temp }}
+        BB_VERBOSITY: 5
       run: |
         coverage run -p `which benchbuild` bootstrap -s
         coverage run -p `which benchbuild` config view
         coverage run -p `which benchbuild` experiment view
         coverage run -p `which benchbuild` project view
-        coverage run -p `which benchbuild` -vvvvv run --full test
-        coverage run -p `which benchbuild` -vvvvv slurm -E empty test
-        coverage run -p `which benchbuild` -vvvvv slurm -E raw bzip2/benchbuild -- container --runroot /foo/bar
+        coverage run -p `which benchbuild` run --full test
+        coverage run -p `which benchbuild` slurm -E empty test
+        coverage run -p `which benchbuild` slurm -E raw bzip2/benchbuild -- container --runroot /foo/bar
+        coverage run -p `which benchbuild` run -E raw bzip2/benchbuild
+        coverage run -p `which benchbuild` container run -E raw bzip2/benchbuild
         coverage run -p `which benchbuild` container bases --export -E raw bzip2/benchbuild
         coverage run -p `which benchbuild` container bases --import -E raw bzip2/benchbuild
-        coverage run -p `which benchbuild` container run -E raw bzip2/benchbuild
         coverage run -p `which benchbuild` container rmi --with-projects -E raw bzip2/benchbuild
         coverage combine
+        coverage report -m
 
     - uses: actions/upload-artifact@master
       with:
@@ -143,12 +152,12 @@ jobs:
         python-version: ["3.10"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -161,14 +170,4 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install .
-        pip install -r requirements.txt
-        pip install -r doc-requirements.txt
-
-    - name: Build documentation
-      run: |
-        mkdocs build
-
-    - name: Deploy documentation
-      if: ${{ github.event_name == 'push' }}
-      run: |
-        mkdocs gh-deploy
+        pip install -r docs/requirements.txt

.github/workflows/docs.yml

@@ -0,0 +1,148 @@
+name: sphinx
+on: [push, pull_request]
+
+env:
+  DEFAULT_BRANCH: "master"
+  #SPHINXOPTS: "-W --keep-going -T"
+  # ^-- If these SPHINXOPTS are enabled, then be strict about the builds and fail on any warnings
+
+jobs:
+  build-and-deploy:
+    name: Build and gh-pages
+    runs-on: ubuntu-latest
+    steps:
+      # https://github.com/marketplace/actions/checkout
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+
+      # https://github.com/marketplace/actions/setup-python
+      # ^-- This gives info on matrix testing.
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      # https://docs.github.com/en/actions/guides/building-and-testing-python#caching-dependencies
+      # ^-- How to set up caching for pip on Ubuntu
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      # https://docs.github.com/en/actions/guides/building-and-testing-python#installing-dependencies
+      # ^-- This gives info on installing dependencies with pip
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r docs/requirements.txt
+          pip install .
+
+      - name: Debugging information
+        run: |
+          echo "github.ref:" ${{github.ref}}
+          echo "github.event_name:" ${{github.event_name}}
+          echo "github.head_ref:" ${{github.head_ref}}
+          echo "github.base_ref:" ${{github.base_ref}}
+          set -x
+          git rev-parse --abbrev-ref HEAD
+          git branch
+          git branch -a
+          git remote -v
+          python -V
+          pip list --not-required
+          pip list
+
+      # Build
+      - uses: ammaraskar/sphinx-problem-matcher@master
+      - name: Build Sphinx docs
+        run: |
+          make -Cdocs dirhtml
+          # This fixes broken copy button icons, as explained in
+          #   https://github.com/coderefinery/sphinx-lesson/issues/50
+          #   https://github.com/executablebooks/sphinx-copybutton/issues/110
+          # This can be removed once these PRs are accepted (but the
+          # fixes also need to propagate to other themes):
+          #   https://github.com/sphinx-doc/sphinx/pull/8524
+          #   https://github.com/readthedocs/sphinx_rtd_theme/pull/1025
+          sed -i 's/url_root="#"/url_root=""/' docs/build/dirhtml/index.html || true
+      # The following supports building all branches and combining on
+      # gh-pages
+
+      # Clone and set up the old gh-pages branch
+      - name: Clone old gh-pages
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          set -x
+          git fetch
+          ( git branch gh-pages remotes/origin/gh-pages && git clone . --branch=gh-pages _gh-pages/ ) || mkdir _gh-pages
+          rm -rf _gh-pages/.git/
+          mkdir -p _gh-pages/branch/
+
+      # If a push and default branch, copy build to _gh-pages/ as the "main"
+      # deployment.
+      - name: Copy new build (default branch)
+        if: |
+          contains(github.event_name, 'push') &&
+          contains(github.ref, env.DEFAULT_BRANCH)
+        run: |
+          set -x
+          # Delete everything under _gh-pages/ that is from the
+          # primary branch deployment.  Eicludes the other branches
+          # _gh-pages/branch-* paths, and not including
+          # _gh-pages itself.
+          find _gh-pages/ -mindepth 1 ! -path '_gh-pages/branch*' -delete
+          rsync -a docs/build/dirhtml/ _gh-pages/
+
+      # If a push and not on default branch, then copy the build to
+      # _gh-pages/branch/$brname (transforming '/' into '--')
+      - name: Copy new build (branch)
+        if: |
+          contains(github.event_name, 'push') &&
+          !contains(github.ref, env.DEFAULT_BRANCH)
+        run: |
+          set -x
+          #brname=$(git rev-parse --abbrev-ref HEAD)
+          brname="${{github.ref}}"
+          brname="${brname##refs/heads/}"
+          brdir=${brname//\//--}   # replace '/' with '--'
+          rm -rf   _gh-pages/branch/${brdir}
+          rsync -a docs/build/dirhtml/ _gh-pages/branch/${brdir}
+
+      # Go through each branch in _gh-pages/branch/, if it's not a
+      # ref, then delete it.
+      - name: Delete old feature branches
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          set -x
+          for brdir in `ls _gh-pages/branch/` ; do
+              brname=${brdir//--/\/}   # replace '--' with '/'
+              if ! git show-ref remotes/origin/$brname ; then
+                  echo "Removing $brdir"
+                  rm -r _gh-pages/branch/$brdir/
+              fi
+          done
+
+      # Add the .nojekyll file
+      - name: nojekyll
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          touch _gh-pages/.nojekyll
+
+      # Deploy
+      # https://github.com/peaceiris/actions-gh-pages
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.event_name == 'push' }}
+        #if: ${{ success() && github.event_name == 'push' && github.ref == 'refs/heads/$defaultBranch' }}
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: _gh-pages/
+          force_orphan: true

.github/workflows/reviewdog.yml

@@ -5,12 +5,12 @@ jobs:
     name: reviewdog
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: "3.10"
 
     - name: Install dependencies
       run: |

.isort.cfg

@@ -1,5 +1,5 @@
 [settings]
-known_third_party = attr,dill,faker,git,jinja2,migrate,mock,parse,pathos,pkg_resources,plumbum,psutil,pygit2,pygtrie,pyparsing,pytest,pytest_git,result,rich,schema,setuptools,six,sqlalchemy,yaml
+known_third_party = attr,dill,faker,git,jinja2,mock,parse,pkg_resources,plumbum,psutil,pygtrie,pyparsing,pytest,pytest_git,result,rich,schema,setuptools,six,sqlalchemy,yaml
 multi_line_output=3
 use_parentheses = True
 include_trailing_comma: True

.pre-commit-config.yaml

@@ -20,6 +20,10 @@ repos:
     hooks:
       - id: isort
         args: ['-nis']
+-   repo: https://github.com/MarcoGorelli/auto-walrus
+    rev: v0.2.2
+    hooks:
+    -   id: auto-walrus
 -   repo: https://github.com/pre-commit/mirrors-yapf
     rev: 'v0.32.0'
     hooks:

.pylintrc

@@ -263,4 +263,4 @@ int-import-graph=
 
 [EXCEPTIONS]
 
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

benchbuild/__init__.py

@@ -28,8 +28,8 @@
 
 def __init__() -> None:
     """Initialize all plugins and settings."""
-    __PLUGINS__.discover()
-    __SETTINGS__.setup_config(CFG)
+    if __PLUGINS__.discover():
+        __SETTINGS__.setup_config(CFG)
 
 
 __init__()

benchbuild/command.py

@@ -2,9 +2,9 @@
 import shutil
 import sys
 import typing as tp
-from collections.abc import Set
 from contextlib import contextmanager
 from pathlib import Path
+from typing import Protocol, runtime_checkable
 
 from plumbum import local
 from plumbum.commands.base import BoundEnvCommand
@@ -18,11 +18,6 @@
 if tp.TYPE_CHECKING:
     import benchbuild.project.Project  # pylint: disable=unused-import
 
-if sys.version_info <= (3, 8):
-    from typing_extensions import Protocol, runtime_checkable
-else:
-    from typing import Protocol, runtime_checkable
-
 LOG = logging.getLogger(__name__)
 
 
@@ -159,8 +154,7 @@ def rendered(
             LOG.error("Cannot render a source directory without a project.")
             return Path(self.unrendered)
 
-        src_path = project.source_of(self.local)
-        if src_path:
+        if (src_path := project.source_of(self.local)):
             return Path(src_path)
         return Path(project.builddir) / self.local
 
@@ -516,8 +510,8 @@ def env(self, **kwargs: str) -> None:
         self._env.update(kwargs)
 
     @property
-    def label(self) -> tp.Optional[str]:
-        return self._label
+    def label(self) -> str:
+        return self._label if self._label else self.name
 
     @label.setter
     def label(self, new_label: str) -> None:
@@ -594,7 +588,7 @@ def __str__(self) -> str:
             command_str = f"{env_str} {command_str}"
         if args_str:
             command_str = f"{command_str} {args_str}"
-        if self.label:
+        if self._label:
             command_str = f"{self._label} {command_str}"
         return command_str
 
@@ -647,12 +641,6 @@ def __str__(self) -> str:
 
 
 def _is_relative_to(p: Path, other: Path) -> bool:
-    if sys.version_info < (3, 9):
-        try:
-            p.relative_to(other)
-            return True
-        except ValueError:
-            return False
     return p.is_relative_to(other)
 
 
@@ -775,6 +763,6 @@ def filter_workload_index(
     This removes all command lists from the index not matching `only`.
     """
 
-    keys = {k for k in index if k and ((only and (k & only)) or (not only))}
+    keys = [k for k in index if k and ((only and (k & only)) or (not only))]
     for k in keys:
         yield index[k]