Merge pull request #788 from google/py-improve-gh-tests

python: improve GH testing workflows

.github/workflows/python-test-published-package.yml

@@ -0,0 +1,39 @@
+# This routinely checks that published packages are installable and work
+# properly. This makes sure that a new version of one of our dependencies is not
+# breaking our releases.
+# TODO: test more magika package versions
+# TODO: check the actual predicted content types
+name: Python - test published packages
+
+on:
+  schedule:
+    - cron: '42 4 * * *' # Run daily
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  unit-testing:
+    strategy:
+      matrix:
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # pin@v5
+        with:
+          python-version: '${{ matrix.python-version }}'
+
+      - name: Install magika
+        run: python3 -m pip install magika
+      - run: python3 -c 'import magika; m = magika.Magika(); print(m)'
+      - run: magika --version
+      # The latest published model does not necessarily support detection for
+      # all types in our tests data; thus, for now we just check that the magika
+      # CLI does not crash when scanning the files, without checking the actual
+      # predictions.
+      - run: magika -r tests_data/basic

.github/workflows/python-test-suite.yml

@@ -22,9 +22,8 @@ jobs:
   unit-testing:
     strategy:
       matrix:
-        python-version: [ "3.8.x", "3.9.x", "3.10.x", "3.11.x", "3.12.x" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
         os: [ "ubuntu-latest", "macos-latest" ]
-        # TODO: add windows, ubuntu:20.04
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4
@@ -39,38 +38,41 @@ jobs:
 
       - name: Install all projects dependencies (with the requested python version)
         working-directory: python
-        run: uv sync --python $(which python3) --all-extras --dev
+        run: uv sync --python ${{ matrix.python-version }} --all-extras --dev
+
+      - name: Print python version
+        working-directory: python
+        run: uv run --python ${{ matrix.python-version }} python --version
 
       - name: Run ruff check
         working-directory: python
-        run: uv run --python $(which python3) ruff check --verbose
+        run: uv run --python ${{ matrix.python-version }} ruff check --verbose
 
       - name: Run ruff format --check
         working-directory: python
-        run: uv run --python $(which python3) ruff format --check --verbose
+        run: uv run --python ${{ matrix.python-version }} ruff format --check --verbose
 
       - name: Run mypy
         working-directory: python
-        run: uv run --python $(which python3) mypy src/magika tests
+        run: uv run --python ${{ matrix.python-version }} mypy src/magika tests
 
       - name: Run the python tests suite
         working-directory: python
-        run: uv run --python $(which python3) pytest tests -m "not slow"
+        run: uv run --python ${{ matrix.python-version }} pytest tests -m "not slow"
 
       - name: Run magika --version
         working-directory: python
-        run: uv run magika --version
+        run: uv run --python ${{ matrix.python-version }} magika --version
         shell: bash # Allows for cross-platform
 
       - name: Run magika with tests_data
         working-directory: python
-        # TODO(https://github.com/google/magika/issues/780): Remove "grep || exit 1" when fixed.
-        run: '(uv run magika -r ../tests_data/basic | grep "code\.asm.*Assembly") || exit 1'
+        run: uv run --python ${{ matrix.python-version }} magika -r ../tests_data/basic
 
       - name: Run "magika cli" quick tests
         working-directory: python
-        run: uv run scripts/run_quick_test_magika_cli.py
+        run: uv run --python ${{ matrix.python-version }} scripts/run_quick_test_magika_cli.py
 
       - name: Run "magika module" quick tests
         working-directory: python
-        run: uv run scripts/run_quick_test_magika_module.py
+        run: uv run --python ${{ matrix.python-version }} scripts/run_quick_test_magika_module.py

python/scripts/run_quick_test_magika_cli.py

@@ -33,16 +33,22 @@
 @click.command()
 def main() -> None:
     basic_tests_dir = (
-        Path(__file__).parent.parent.parent / "tests_data" / "basic"
-    ).resolve()
+        Path(__file__).resolve().parent.parent.parent / "tests_data" / "basic"
+    )
+    assert basic_tests_dir.is_dir()
 
     p = subprocess.run(
         ["magika", "-r", "--label", str(basic_tests_dir)],
         capture_output=True,
-        check=True,
         text=True,
     )
 
+    if p.returncode != 0:
+        print("ERROR: magika CLI exited with non-zero status.")
+        print(f"stdout:\n{p.stdout}\n" + "-" * 40)
+        print(f"stderr:\n{p.stderr}\n" + "-" * 40)
+        sys.exit(1)
+
     assert p.stderr == ""
 
     with_error = False