ci(python): Refactor Python release workflow (pola-rs#11382)

cmdlineluser · Sep 28, 2023 · 8f0fae4 · 8f0fae4
1 parent 7c2098c
commit 8f0fae4
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 158 deletions.
diff --git a/.github/deploy_manylinux.sh b/.github/deploy_manylinux.sh
diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
@@ -8,73 +8,52 @@ on:
 env:
   RUST_TOOLCHAIN: nightly-2023-08-26
   PYTHON_VERSION: '3.8'
-  MATURIN_VERSION: '1.2.1'
+  MATURIN_VERSION: '1.2.3'
   MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
 
 defaults:
   run:
     shell: bash
 
 jobs:
-  manylinux-x64_64:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Fix README symlink
-        run: |
-          rm py-polars/README.md
-          cp README.md py-polars/README.md
-
-      - name: Publish wheel
-        uses: PyO3/maturin-action@v1
-        env:
-          RUSTFLAGS: -C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma
-        with:
-          command: publish
-          args: -m py-polars/Cargo.toml --skip-existing -o wheels -u ritchie46
-          maturin-version: ${{ env.MATURIN_VERSION }}
-          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
+  publish-wheels:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            cpu: x86-64
+            sdist: true
+          - os: ubuntu-latest
+            cpu: aarch64
+          - os: macos-latest
+            cpu: x86-64
+          - os: macos-latest
+            cpu: aarch64
+          - os: windows-32gb-ram
+            cpu: x86-64
+          - os: ubuntu-latest
+            cpu: x86-64
+            lts-cpu: true
+            sdist: true
+          - os: ubuntu-latest
+            cpu: x86-64
+            u64-idx: true
+            sdist: true
 
-  # Needed for Docker on Apple M1
-  manylinux-aarch64:
-    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
 
-      # Needed to avoid out-of-memory error
-      - name: Set Swap Space
+      # Avoid potential out-of-memory errors
+      - name: Set swap space for Linux
+        if: matrix.os == 'ubuntu-latest'
         uses: pierotofy/set-swap-space@master
         with:
           swap-size-gb: 10
 
-      - name: Fix README symlink
-        run: |
-          rm py-polars/README.md
-          cp README.md py-polars/README.md
-
-      - name: Publish wheel
-        uses: PyO3/maturin-action@v1
-        env:
-          JEMALLOC_SYS_WITH_LG_PAGE: 16
-        with:
-          command: publish
-          args: -m py-polars/Cargo.toml --skip-existing --no-sdist -o wheels -i python -u ritchie46
-          target: aarch64-unknown-linux-gnu
-          maturin-version: ${{ env.MATURIN_VERSION }}
-          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
-
-  manylinux-bigidx:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
@@ -83,95 +62,43 @@ jobs:
           rm py-polars/README.md
           cp README.md py-polars/README.md
 
-      - name: Prepare bigidx
+      - name: Prepare lts-cpu
+        if: matrix.lts-cpu
+        run: sed -i 's/name = "polars"/name = "polars-lts-cpu"/' py-polars/pyproject.toml
+
+      - name: Prepare u64-idx
+        if: matrix.u64-idx
         run: |
           sed -i 's/name = "polars"/name = "polars-u64-idx"/' py-polars/pyproject.toml
           # A brittle hack to insert the 'bigidx' feature
           sed -i 's/"dynamic_group_by",/"dynamic_group_by",\n"bigidx",/' py-polars/Cargo.toml
 
-      - name: Publish wheel
-        uses: PyO3/maturin-action@v1
-        env:
-          RUSTFLAGS: -C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma
-        with:
-          command: publish
-          args: -m py-polars/Cargo.toml --skip-existing -o wheels -u ritchie46
-          maturin-version: ${{ env.MATURIN_VERSION }}
-          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Set RUSTFLAGS for x86-64
+        if: matrix.cpu == 'x86-64' && !matrix.lts-cpu
+        run: echo "RUSTFLAGS=-C target-feature=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma" >> $GITHUB_ENV
 
-  manylinux-x64_64-lts-cpu:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Set RUSTFLAGS for x86-64 LTS CPU
+        if: matrix.cpu == 'x86-64' && matrix.lts-cpu
+        run: echo "RUSTFLAGS=-C target-feature=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt --cfg use_mimalloc" >> $GITHUB_ENV
 
-      - name: Fix README symlink
+      - name: Set Rust target for aarch64
+        if: matrix.cpu == 'aarch64'
+        id: target
         run: |
-          rm py-polars/README.md
-          cp README.md py-polars/README.md
-
-      - name: Prepare lts-cpu
-        run: sed -i 's/name = "polars"/name = "polars-lts-cpu"/' py-polars/pyproject.toml
-
-      - name: Publish wheel
-        uses: PyO3/maturin-action@v1
-        env:
-          RUSTFLAGS: -C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt --cfg use_mimalloc
-        with:
-          command: publish
-          args: -m py-polars/Cargo.toml --skip-existing -o wheels -u ritchie46
-          maturin-version: ${{ env.MATURIN_VERSION }}
-          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
-
-  win-macos:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, windows-32gb-ram]
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          TARGET=${{ matrix.os == 'macos-latest' && 'aarch64-apple-darwin' || 'aarch64-unknown-linux-gnu'}}
+          echo "target=$TARGET" >> $GITHUB_OUTPUT
 
-      - name: Fix README symlink
+      - name: Set jemalloc for aarch64 Linux
+        if: matrix.cpu == 'aarch64' && matrix.os == 'ubuntu-latest'
         run: |
-          rm py-polars/README.md
-          cp README.md py-polars/README.md
+          echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> $GITHUB_ENV
 
       - name: Publish wheel
         uses: PyO3/maturin-action@v1
-        env:
-          RUSTFLAGS: -C target-feature=+fxsr,+sse,+sse2,+sse3,+sse4.1,+sse4.2
         with:
           command: publish
-          args: -m py-polars/Cargo.toml --no-sdist --skip-existing -o wheels -i python -u ritchie46
+          target: ${{ steps.target.outputs.target }}
+          working-directory: py-polars
+          args: --skip-existing ${{ !matrix.sdist && '--no-sdist' || ''}}
           maturin-version: ${{ env.MATURIN_VERSION }}
           rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
-
-  macos-aarch64:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Fix README symlink
-        run: |
-          rm py-polars/README.md
-          cp README.md py-polars/README.md
-
-      - name: Set up Rust targets
-        run: rustup target add aarch64-apple-darwin
-
-      - name: Publish wheel
-        uses: PyO3/maturin-action@v1
-        with:
-          command: publish
-          args: -m py-polars/Cargo.toml --target aarch64-apple-darwin --no-sdist -o wheels -i python -u ritchie46
-          maturin-version: ${{ env.MATURIN_VERSION }}