Adding Python monorepo builder

.github/workflows/test_and_deploy.yml

@@ -18,15 +18,6 @@ concurrency:
 jobs:
   test-and-build-dpf-ssw-aligner:
     name: "Build & Test dpf-ssw-aligner-pyrs"
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, windows-2019, macos-11]
-        project_folder: ["./src/python/dpf-ssw-aligner-rs"]
-        python-version: ['cp310', 'cp311']
-    runs-on: ${{ matrix.os }}
-    env:
-      CIBW_ENVIRONMENT: 'PATH="$PATH:$HOME/.cargo/bin"'
-      CIBW_SKIP: "p*-win* *-win32 *-win_arm64 *-musllinux_*"
 
     steps:
       - uses: actions/checkout@v3
@@ -39,65 +30,31 @@ jobs:
         with:
           platforms: all
 
-      - name: Install latest nightly
-        uses: dtolnay/rust-toolchain@nightly
-
-      - name: Build & test wheels dpf-ssw-aligner-rs wheels
-        uses: pypa/[email protected]
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
         with:
-          package-dir: src/python/dpf-ssw-aligner-rs
-        env:
-          CIBW_BUILD: '${{ matrix.python-version }}-*'
-          # we build for "alt_arch_name" if it exists, else 'auto'
-          CIBW_ARCHS: ${{ matrix.alt_arch_name || 'auto' }}
-          CIBW_ENVIRONMENT: 'PATH="$HOME/.cargo/bin:$PATH" CARGO_TERM_COLOR="always"'
-          CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"'
-          CIBW_BEFORE_ALL_WINDOWS: "rustup target add x86_64-pc-windows-msvc i686-pc-windows-msvc"
-          CIBW_BEFORE_ALL_MACOS: "rustup target add aarch64-apple-darwin x86_64-apple-darwin"
-          CIBW_BEFORE_BUILD: rustup show
-          CIBW_BEFORE_BUILD_LINUX: >
-            curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=nightly --profile=minimal -y &&
-            rustup show
-          # CIBW_TEST_SKIP: '*-macosx_universal2:arm64'
-          CIBW_BUILD_VERBOSITY: 1
+          python-version: "3.12"
 
-      - name: Build sdist
-        run: pipx run build --sdist --outdir wheelhouse src/python/dpf-ssw-aligner-rs
+      - name: Build, Test, and Upload dpf-ssw-aligner-rs
+        run: python ./mono.py cicd_dpf_ssw_aligner
 
       - uses: actions/upload-artifact@v3
         with:
           path: ./wheelhouse/dpf_ssw_aligner*.whl
 
   test-and-build-dpf-mrcfile:
     name: "Build & Test dpf-mrcfile"
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, windows-2019, macos-11]
-        project_folder: ["./src/python/dpf-mrcfile"]
-        python-version: ['3.10', '3.11']
-    runs-on: ${{ matrix.os }}
-    env:
-      CIBW_SKIP: "p*-win* *-win32 *-win_arm64 *-musllinux_*"
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pytest build src/python/dpf-mrcfile
-
-      - name: Test with pytest
-        run: |
-          pytest src/python/dpf-mrcfile/test
+          python-version: "3.12"
 
-      - name: Build sdist
-        run: python -m build -o wheelhouse src/python/dpf-mrcfile
+      - name: Build, Test, and Upload dpf-mrcfile
+        run: python ./mono.py cicd_dpf_ssw_aligner
 
       - uses: actions/upload-artifact@v3
         if: runner.os == 'Linux'

mono.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+"""
+For now we'll run most with shell=True so that we can move away from this quickly if necessary
+"""
+
+import argparse
+import os
+import platform
+from typing import List
+import sys
+import logging
+import subprocess
+
+LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper()
+logging.basicConfig(level=LOGLEVEL)
+log = logging.getLogger(__name__)
+
+CIBUILDWHEEL_VERSION = "2.16.2"
+CIBUILDWHEEL_PLATFORM = "auto"
+
+
+def parseargs(args: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--mode",
+        choices=["install_ci_deps", "cicd_dpf_ssw_aligner_rs", "cicd_dpf_mrcfile"],
+        help="mode to run",
+        required=True,
+    )
+    parser.add_argument(
+        "--cibw_platform",
+        choices=["auto", "linux", "macos", "windows"],
+        help="cibuildwheel platform to build",
+        default=CIBUILDWHEEL_PLATFORM,
+    )
+    return parser.parse_args(args)
+
+
+def run_l(cmd: list[str]) -> None:
+    str_cmd = " ".join(cmd)
+    log.debug(f"Running: {str_cmd}")
+    ret = subprocess.run(cmd)
+    if ret.returncode:
+        raise RuntimeError(f"Failure with command: {str_cmd}")
+
+
+def run_s(cmd: str, env_add: dict[str, str]) -> None:
+    log.debug(f"Running: {cmd}")
+    env = dict(os.environ)
+    env.update(env_add)
+    ret = subprocess.run(cmd, env=env, shell=True)
+    if ret.returncode:
+        raise RuntimeError(f"Failure with command: {cmd}")
+
+
+def install_rust_cibw_command():
+    return (
+        "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs"
+        " | sh -s -- --default-toolchain none -y"
+        " && rustup toolchain install nightly --allow-downgrade --profile minimal --component clippy"
+        " && rustup show"
+    )
+
+
+def install_ci_deps():
+    cmds = f"""
+python -m pip install cibuildwheel=={CIBUILDWHEEL_VERSION}
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain none -y
+rustup toolchain install nightly --allow-downgrade --profile minimal --component clippy
+rustup show
+pip install pipx
+"""
+    match platform.system():
+        case "Darwin":
+            cmds += "rustup target add aarch64-apple-darwin x86_64-apple-darwin"
+        case "Windows":
+            cmds += "rustup target add x86_64-pc-windows-msvc i686-pc-windows-msvc"
+    for cmd in cmds.split("\n"):
+        if cmd:
+            run_s(cmd, {})
+
+
+def cicd_dpf_ssw_aligner_rs():
+    env = {
+        "CIBW_PLATFORM": CIBUILDWHEEL_PLATFORM,
+        "CIBW_ARCHS": "auto",
+        "CIBW_ENVIRONMENT": 'PATH="$HOME/.cargo/bin:$PATH" CARGO_TERM_COLOR="always"',
+        "CIBW_ENVIRONMENT_WINDOWS": 'PATH="$UserProfile\\.cargo\\bin;$PATH"',
+        "CIBW_BEFORE_BUILD": install_rust_cibw_command(),
+        "CIBW_BUILD_VERBOSITY": "1",
+    }
+
+    cmds = """
+python -m cibuildwheel ./src/python/dpf-ssw-aligner-rs --output-dir wheelhouse
+ls wheelhouse
+pipx run build --sdist --outdir wheelhouse ./src/python/dpf-ssw-aligner-rs
+ls wheelhouse
+    """
+    for cmd in cmds.split("\n"):
+        if cmd:
+            run_s(cmd, env)
+
+
+def cicd_dpf_mrcfile():
+    env = {
+        "CIBW_PLATFORM": CIBUILDWHEEL_PLATFORM,
+        "CIBW_ARCHS": "auto",
+        "CIBW_ENVIRONMENT": 'PATH="$HOME/.cargo/bin:$PATH" CARGO_TERM_COLOR="always"',
+        "CIBW_ENVIRONMENT_WINDOWS": 'PATH="$UserProfile\\.cargo\\bin;$PATH"',
+        "CIBW_BEFORE_BUILD": install_rust_cibw_command(),
+        "CIBW_BUILD_VERBOSITY": "1",
+    }
+
+    cmds = """
+python -m cibuildwheel ./src/python/dpf-mrcfile --output-dir wheelhouse
+ls wheelhouse
+pipx run build --sdist --outdir wheelhouse ./src/python/dpf-mrcfile
+ls wheelhouse
+    """
+    for cmd in cmds.split("\n"):
+        if cmd:
+            run_s(cmd, env)
+
+
+def main(_args: List[str]) -> None:
+    args = parseargs(_args)
+    global CIBUILDWHEEL_PLATFORM
+    CIBUILDWHEEL_PLATFORM = args.cibw_platform
+    match args.mode:
+        case "install_ci_deps":
+            install_ci_deps()
+        case "cicd_dpf_ssw_aligner_rs":
+            install_ci_deps()
+            cicd_dpf_ssw_aligner_rs()
+        case "cicd_dpf_mrcfile":
+            install_ci_deps()
+            cicd_dpf_mrcfile()
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

src/python/dpf-mrcfile/pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-name = "dpf_mrcfile"
+name = "dpf-mrcfile"
 description="A library for handling scientific volume and image-stack data like from .MAP .MRC .MRCS"
 requires-python = ">=3.10"
 authors = [{name = "Danny Farrell", email = "[email protected]"}]
@@ -14,6 +14,48 @@ classifiers = [
     "Operating System :: MacOS :: MacOS X",
 ]
 
+
+[project.optional-dependencies]
+dev = ["pytest>=6.0", "twine"]
+
 [tool.cibuildwheel]
 test-requires = "pytest"
 test-command = "pytest {package}/test"
+test-skip = ["*universal2:arm64"]
+skip = "*-win32 *-win_arm64 pp*win* *musllinux*"
+
+[tool.ruff]
+src = ["src/dpf-mrcfile"]
+
+[tool.ruff.lint]
+extend-select = [
+  "B",      # flake8-bugbear
+  "I",      # isort
+  "ARG",    # flake8-unused-arguments
+  "C4",     # flake8-comprehensions
+  "EM",     # flake8-errmsg
+  "ICN",    # flake8-import-conventions
+  "G",      # flake8-logging-format
+  "PGH",    # pygrep-hooks
+  "PIE",    # flake8-pie
+  "PL",     # pylint
+  "PTH",    # flake8-use-pathlib
+  "RET",    # flake8-return
+  "RUF",    # Ruff-specific
+  "SIM",    # flake8-simplify
+  "T20",    # flake8-print
+  "UP",     # pyupgrade
+  "YTT",    # flake8-2020
+  "EXE",    # flake8-executable
+  "NPY",    # NumPy specific rules
+  "PD",     # pandas-vet
+]
+ignore = [
+  "PLR",    # Design related pylint codes
+  "PT",     # flake8-pytest-style
+]
+isort.required-imports = ["from __future__ import annotations"]
+
+[tool.ruff.per-file-ignores]
+"src/dpf-mrcfile/test/**" = ["T20"]
+