Merge pull request PolusAI#196 from JesseMckinzie/cuda_wheels_ci_cd_v2

Add CUDA wheels build and PyPI publishing to Github actions
sameeul · Jan 29, 2024 · 13a04cc · 13a04cc
2 parents 8c33437 + adfb6d0
commit 13a04cc
Show file tree

Hide file tree

Showing 10 changed files with 295 additions and 13 deletions.
diff --git a/.github/workflows/build_cuda11_wheels.yml b/.github/workflows/build_cuda11_wheels.yml
@@ -0,0 +1,60 @@
+name: Build CUDA 11 Wheels
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+        cibw_archs: ["auto64"]
+        cibw_build: ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
+
+    steps:
+      - uses: actions/checkout@v3
+        name: Check out
+        with:
+          submodules: recursive    
+
+      - uses: ilammy/msvc-dev-cmd@v1
+        name: Add MSVS Path
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==2.16.2 delvewheel wheel
+
+      - name: Building wheels 
+        run: |
+          python -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: ${{ matrix.cibw_build }}
+          CIBW_SKIP: "*musllinux*"
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_BEFORE_ALL_LINUX:  yum install -y llvm libevent-devel openssl-devel &&
+                                   bash ci-utils/install_cuda11_yum.sh && 
+                                   bash ci-utils/install_arrow_yum.sh &&
+                                   bash ci-utils/install_prereq_linux.sh --build_arrow no &&
+                                   mkdir -p /tmp/nyxus_bld &&
+                                   cp -r local_install /tmp/nyxus_bld
+          CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" 
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.10 --exclude=libcufft.so.10.4.2.109 --exclude=libcudart.so --exclude=libcudart.so.11.0 --exclude=libcudart.so.11.3.109  -w {dest_dir} {wheel}"
+          CIBW_ARCHS: ${{ matrix.cibw_archs }}
+          CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
+          CIBW_TEST_COMMAND: pytest {project}/tests/python -m "not skip_ci"
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: nyxus-cuda11-wheels
+          path: dist/*.whl
+          retention-days: 1
diff --git a/.github/workflows/build_cuda12_wheels.yml b/.github/workflows/build_cuda12_wheels.yml
@@ -0,0 +1,60 @@
+name: Build CUDA 12 Wheels
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+        cibw_archs: ["auto64"]
+        cibw_build: ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
+
+    steps:
+      - uses: actions/checkout@v3
+        name: Check out
+        with:
+          submodules: recursive    
+
+      - uses: ilammy/msvc-dev-cmd@v1
+        name: Add MSVS Path
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==2.16.2 delvewheel wheel
+
+      - name: Building wheels 
+        run: |
+          python -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: ${{ matrix.cibw_build }}
+          CIBW_SKIP: "*musllinux*"
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_BEFORE_ALL_LINUX:  yum install -y llvm libevent-devel openssl-devel &&
+                                   bash ci-utils/install_cuda12_yum.sh && 
+                                   bash ci-utils/install_arrow_yum.sh &&
+                                   bash ci-utils/install_prereq_linux.sh --build_arrow no &&
+                                   mkdir -p /tmp/nyxus_bld &&
+                                   cp -r local_install /tmp/nyxus_bld
+          CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" 
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.11 --exclude=libcufft.so.11.0.12.1  --exclude=libcudart.so --exclude=libcudart.so.12 --exclude=libcudart.so.12.3.101  -w {dest_dir} {wheel}
+          CIBW_ARCHS: ${{ matrix.cibw_archs }}
+          CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
+          CIBW_TEST_COMMAND: pytest {project}/tests/python -m "not skip_ci"
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: nyxus-cuda12-wheels
+          path: dist/*.whl
+          retention-days: 1
diff --git a/.github/workflows/publish_cuda11_pypi.yml b/.github/workflows/publish_cuda11_pypi.yml
@@ -0,0 +1,65 @@
+name: Publish CUDA 11 to PyPi
+
+on:
+  release:
+    types: [published]
+
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+        cibw_archs: ["auto64"]
+        cibw_build: ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
+
+    steps:
+      - uses: actions/checkout@v3
+        name: Check out
+        with:
+          submodules: recursive    
+
+      - uses: ilammy/msvc-dev-cmd@v1
+        name: Add MSVS Path
+
+      - uses: actions/setup-python@v4
+        name: Install PythonF
+        with:
+          python-version: '3.9'
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==2.16.2 delvewheel wheel
+
+      - name: Building wheels 
+        run: |
+          python -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: ${{ matrix.cibw_build }}
+          CIBW_SKIP: "*musllinux*"
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_BEFORE_ALL_LINUX:  yum install -y llvm libevent-devel openssl-devel &&
+                                   bash ci-utils/install_cuda11_yum.sh && 
+                                   bash ci-utils/install_arrow_yum.sh &&
+                                   bash ci-utils/install_prereq_linux.sh --build_arrow no &&
+                                   mkdir -p /tmp/nyxus_bld &&
+                                   cp -r local_install /tmp/nyxus_bld
+          CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" 
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.10 --exclude=libcufft.so.10.4.2.109 --exclude=libcudart.so --exclude=libcudart.so.11.0 --exclude=libcudart.so.11.3.109  -w {dest_dir} {wheel}"
+          CIBW_ARCHS: ${{ matrix.cibw_archs }}
+          CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
+          CIBW_TEST_COMMAND: pytest {project}/tests/python -m "not skip_ci"
+
+      - name: Install Dependencies
+        run: python -m pip install --upgrade twine requests
+
+      - name: Publish to PyPi
+        run: python -m twine upload dist/*.whl
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TWINE_API_KEY }}
+          TWINE_REPOSITORY: pypi
diff --git a/.github/workflows/publish_cuda12_pypi.yml b/.github/workflows/publish_cuda12_pypi.yml
@@ -0,0 +1,65 @@
+name: Publish CUDA 12 to PyPi
+
+on:
+  release:
+    types: [published]
+
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+        cibw_archs: ["auto64"]
+        cibw_build: ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
+
+    steps:
+      - uses: actions/checkout@v3
+        name: Check out
+        with:
+          submodules: recursive    
+
+      - uses: ilammy/msvc-dev-cmd@v1
+        name: Add MSVS Path
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==2.16.2 delvewheel wheel
+
+      - name: Building wheels 
+        run: |
+          python -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: ${{ matrix.cibw_build }}
+          CIBW_SKIP: "*musllinux*"
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_BEFORE_ALL_LINUX:  yum install -y llvm libevent-devel openssl-devel &&
+                                   bash ci-utils/install_cuda12_yum.sh && 
+                                   bash ci-utils/install_arrow_yum.sh &&
+                                   bash ci-utils/install_prereq_linux.sh --build_arrow no &&
+                                   mkdir -p /tmp/nyxus_bld &&
+                                   cp -r local_install /tmp/nyxus_bld
+          CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" 
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.11 --exclude=libcufft.so.11.0.12.1  --exclude=libcudart.so --exclude=libcudart.so.12 --exclude=libcudart.so.12.3.101  -w {dest_dir} {wheel}
+          CIBW_ARCHS: ${{ matrix.cibw_archs }}
+          CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
+          CIBW_TEST_COMMAND: pytest {project}/tests/python -m "not skip_ci"
+
+      - name: Install Dependencies
+        run: python -m pip install --upgrade twine requests
+
+      - name: Publish to PyPi
+        run: python -m twine upload dist/*.whl
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TWINE_API_KEY }}
+          TWINE_REPOSITORY: pypi
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -34,13 +34,13 @@ set(Z5_REQUESTED ${USE_Z5})
 set(ARROW_REQUESTED ${USE_ARROW})
 set(DCMTK_REQUESTED ${USE_DCMTK})
 
-option(USEGPU "Use GPU" ON)
+option(USEGPU "Use GPU" OFF)
 set(CUDA_REQUESTED ${USEGPU})
-check_language(CUDA)
-find_package(CUDAToolkit)
 if (USEGPU)
+	check_language(CUDA)
+	find_package(CUDAToolkit)
 	if (NOT(CMAKE_CUDA_COMPILER) OR NOT(CUDAToolkit_FOUND))
-		set(USEGPU OFF)
+		message(FATAL_ERROR "USEGPU flag was set to ON but no CUDA compiler was found. Set USEGPU to OFF to continue without CUDA support.")
 	endif()
 endif()
 
@@ -60,10 +60,6 @@ if(USEGPU)
 	else() # some old CUDA version (<10)
 		set(CUDA_ARCH_LIST "50")
 	endif()
-else()
-	if(CUDA_REQUESTED)
-		message(WARNING "CUDA Toolkit not found. USEGPU flag will be ignored.")
-	endif()
 endif()
 
 #==== Compiler Options

diff --git a/ci-utils/install_cuda11_yum.sh b/ci-utils/install_cuda11_yum.sh
@@ -0,0 +1,6 @@
+yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo &&
+yum clean all
+yum -y install cuda-toolkit-11-3
+ls -al /usr/local
+export PATH=$PATH:/usr/local/cuda/bin
+nvcc --version
diff --git a/ci-utils/install_cuda12_yum.sh b/ci-utils/install_cuda12_yum.sh
@@ -0,0 +1,6 @@
+yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo &&
+yum clean all
+yum -y install cuda-toolkit-12-3
+ls -al /usr/local
+export PATH=$PATH:/usr/local/cuda/bin
+nvcc --version
diff --git a/pytest.ini b/pytest.ini
@@ -1,4 +1,5 @@
 [pytest]
 markers =
     arrow: marks tests as using Apache Arrow (deselect with '-m "not arrow"')
-    serial
+    serial
+    skip_ci: marks tests that need to be skipped in the CI when building GPU wheels with device on the machine
diff --git a/setup.py b/setup.py
@@ -1,7 +1,6 @@
 import os
 import re
 import sys
-import sysconfig
 import versioneer
 import platform
 import subprocess
@@ -87,13 +86,36 @@ def build_extension(self, ext):
         )
         print()  # Add an empty line for cleaner output
 
+
+def get_cuda_major_version():
+    try:
+        output = subprocess.check_output(['nvcc', '--version']).decode('utf-8')
+        match = re.search(r'release (\d+)\.', output)
+        if match:
+            return match.group(1)
+        else:
+            return None
+    except Exception as e:
+        return None    
+
+def get_name():
+    if len(os.environ.get("CMAKE_ARGS", "")):
+        args = os.environ.get("CMAKE_ARGS", "").split(" ")
+        if "-DUSEGPU=ON" in args: #check if gpu build is requested
+            cuda_major_version = get_cuda_major_version()
+            if cuda_major_version is None:
+                raise ValueError("USEGPU flag was set to ON but no CUDA version was found. Set USEGPU=OFF to continue.")
+            else:
+                return f"nyxus-cuda{cuda_major_version}x"
+    return "nyxus"
+
 
 with open("README.md", "r") as fh:
     long_description = fh.read()
-
-
+    
+    
 setup(
-    name="nyxus",
+    name=get_name(),
     version=versioneer.get_version(),
     cmdclass=versioneer.get_cmdclass(dict(build_ext=CMakeBuild)),
     author="Andriy Kharchenko",

diff --git a/tests/python/test_nyxus.py b/tests/python/test_nyxus.py
@@ -86,6 +86,7 @@ def test_featurize_montage(self):
 
             assert len(montage_not_equal) == 0
 
+        @pytest.mark.skip_ci
         def test_gabor_gpu(self):
             # cpu gabor
             cpu_nyx = nyxus.Nyxus(["GABOR"])