From 40517d2cf02883fd647008d0a8739a1e9f457826 Mon Sep 17 00:00:00 2001 From: sameeul Date: Thu, 8 Feb 2024 10:46:32 -0500 Subject: [PATCH] fix apple arm64 build and update gpu wheel build --- .github/workflows/build_cuda11_wheels.yml | 4 ++-- .github/workflows/build_cuda12_wheels.yml | 4 ++-- .github/workflows/build_wheels.yml | 7 ++++--- .github/workflows/publish_cuda11_pypi.yml | 4 ++-- .github/workflows/publish_cuda12_pypi.yml | 4 ++-- setup.py | 17 +++++++++-------- 6 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_cuda11_wheels.yml b/.github/workflows/build_cuda11_wheels.yml index f0ed7831..42159827 100644 --- a/.github/workflows/build_cuda11_wheels.yml +++ b/.github/workflows/build_cuda11_wheels.yml @@ -46,11 +46,11 @@ jobs: bash ci-utils/install_prereq_linux.sh --build_arrow no && mkdir -p /tmp/nyxus_bld && cp -r local_install /tmp/nyxus_bld - CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" + CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" NYXUS_GPU_WHEEL="ON" CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.10 --exclude=libcufft.so.10.4.2.109 --exclude=libcudart.so --exclude=libcudart.so.11.0 --exclude=libcudart.so.11.3.109 -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio - CIBW_TEST_COMMAND: pytest {project}/tests/python -m "not skip_ci" + CIBW_TEST_COMMAND: pytest -vv {project}/tests/python -m "not skip_ci" - name: Upload Artifact uses: actions/upload-artifact@v3 diff --git a/.github/workflows/build_cuda12_wheels.yml b/.github/workflows/build_cuda12_wheels.yml index 1c23f983..809ade22 100644 --- a/.github/workflows/build_cuda12_wheels.yml +++ b/.github/workflows/build_cuda12_wheels.yml @@ -46,11 +46,11 @@ jobs: bash ci-utils/install_prereq_linux.sh --build_arrow no && mkdir -p /tmp/nyxus_bld && cp -r local_install /tmp/nyxus_bld - CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" + CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" NYXUS_GPU_WHEEL="ON" CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.11 --exclude=libcufft.so.11.0.12.1 --exclude=libcudart.so --exclude=libcudart.so.12 --exclude=libcudart.so.12.3.101 -w {dest_dir} {wheel} CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio - CIBW_TEST_COMMAND: pytest {project}/tests/python -m "not skip_ci" + CIBW_TEST_COMMAND: pytest -vv {project}/tests/python -m "not skip_ci" - name: Upload Artifact uses: actions/upload-artifact@v3 diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index a95a5725..b9524900 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -61,7 +61,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio - CIBW_TEST_COMMAND: pytest {project}/tests/python + CIBW_TEST_COMMAND: pytest -vv {project}/tests/python - name: Upload Artifact uses: actions/upload-artifact@v3 @@ -93,7 +93,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==2.16.2 delvewheel wheel + python -m pip install cibuildwheel==2.16.2 delocate wheel - name: Building wheels run: | @@ -103,6 +103,7 @@ jobs: CIBW_BUILD_VERBOSITY: 3 CIBW_ARCHS_MACOS: arm64 CIBW_BEFORE_ALL_MACOS: brew install llvm@16 && + sudo xcode-select -s /Applications/Xcode_14.2.app && bash ci-utils/install_prereq_linux.sh --build_arrow yes && mkdir -p /tmp/nyxus_bld && cp -r local_install /tmp/nyxus_bld @@ -111,7 +112,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio - CIBW_TEST_COMMAND: pytest {project}/tests/python + CIBW_TEST_COMMAND: pytest -vv {project}/tests/python - name: Upload Artifact uses: actions/upload-artifact@v3 diff --git a/.github/workflows/publish_cuda11_pypi.yml b/.github/workflows/publish_cuda11_pypi.yml index b8bbe8e2..d1b9ceeb 100644 --- a/.github/workflows/publish_cuda11_pypi.yml +++ b/.github/workflows/publish_cuda11_pypi.yml @@ -48,11 +48,11 @@ jobs: bash ci-utils/install_prereq_linux.sh --build_arrow no && mkdir -p /tmp/nyxus_bld && cp -r local_install /tmp/nyxus_bld - CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" + CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" NYXUS_GPU_WHEEL="ON" CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.10 --exclude=libcufft.so.10.4.2.109 --exclude=libcudart.so --exclude=libcudart.so.11.0 --exclude=libcudart.so.11.3.109 -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio - CIBW_TEST_COMMAND: pytest {project}/tests/python -m "not skip_ci" + CIBW_TEST_COMMAND: pytest -vv {project}/tests/python -m "not skip_ci" - name: Install Dependencies run: python -m pip install --upgrade twine requests diff --git a/.github/workflows/publish_cuda12_pypi.yml b/.github/workflows/publish_cuda12_pypi.yml index 1d20df9f..3668a977 100644 --- a/.github/workflows/publish_cuda12_pypi.yml +++ b/.github/workflows/publish_cuda12_pypi.yml @@ -48,11 +48,11 @@ jobs: bash ci-utils/install_prereq_linux.sh --build_arrow no && mkdir -p /tmp/nyxus_bld && cp -r local_install /tmp/nyxus_bld - CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" + CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" CPATH="/usr/local/cuda/targets/x86_64-linux/include:$CPATH" PATH="/usr/local/cuda/bin:$PATH" ON_GITHUB="TRUE" NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install" CXXFLAGS="-I /usr/local/cuda/include" CMAKE_ARGS="-DUSEGPU=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES=/usr/local/cuda/include" NYXUS_GPU_WHEEL="ON" CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude=libcufft.so --exclude=libcufft.so.11 --exclude=libcufft.so.11.0.12.1 --exclude=libcudart.so --exclude=libcudart.so.12 --exclude=libcudart.so.12.3.101 -w {dest_dir} {wheel} CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio - CIBW_TEST_COMMAND: pytest {project}/tests/python -m "not skip_ci" + CIBW_TEST_COMMAND: pytest -vv {project}/tests/python -m "not skip_ci" - name: Install Dependencies run: python -m pip install --upgrade twine requests diff --git a/setup.py b/setup.py index 76afc34a..6a6a27fd 100644 --- a/setup.py +++ b/setup.py @@ -99,14 +99,15 @@ def get_cuda_major_version(): return None def get_name(): - if len(os.environ.get("CMAKE_ARGS", "")): - args = os.environ.get("CMAKE_ARGS", "").split(" ") - if "-DUSEGPU=ON" in args: #check if gpu build is requested - cuda_major_version = get_cuda_major_version() - if cuda_major_version is None: - raise ValueError("USEGPU flag was set to ON but no CUDA version was found. Set USEGPU=OFF to continue.") - else: - return f"nyxus-cuda{cuda_major_version}x" + if os.environ.get("NYXUS_GPU_WHEEL", "") == "ON": + if len(os.environ.get("CMAKE_ARGS", "")): + args = os.environ.get("CMAKE_ARGS", "").split(" ") + if "-DUSEGPU=ON" in args: #check if gpu build is requested + cuda_major_version = get_cuda_major_version() + if cuda_major_version is None: + raise RuntimeError("USEGPU flag was set to ON but no CUDA version was found. Set USEGPU=OFF to continue.") + else: + return f"nyxus-cuda{cuda_major_version}x" return "nyxus"