Setup build wheels using cibuildwheel for all systems (#559)

* initial setup of cibuildwheel * skip windows test * revert obsolete change * skip pypy on windows * clean old wheel build configuration * bump cibuildwheel version * fix flake test * limit wheel build run * undo skiping/failing tests * comment win32 steps in build-ocl-windows * use stable ICD on macos * Update scripts/build-ocl-windows.sh Co-authored-by: Andreas Klöckner <[email protected]> * rename file for wheel build * add conda-forge explanation * remove mako from build-system requires * disable mako check for build under cibuildwheel * remove obsolete check of mako * add license information add missed SCRIPT_DIR * try use configure script * musllinux build * Apply suggestions from code review Co-authored-by: Andreas Klöckner <[email protected]> Co-authored-by: Andreas Klöckner <[email protected]>
inducer · May 17, 2022 · 04143b9 · 04143b9
1 parent 1205024
commit 04143b9
Show file tree

Hide file tree

Showing 11 changed files with 174 additions and 159 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -120,43 +120,6 @@ jobs:
                 (cd examples; rm -f gl_*)
                 run_examples --no-require-main
 
-    wheels:
-        name: Build and upload wheels
-        runs-on: ubuntu-latest
-        strategy:
-            matrix:
-                DOCKER_IMAGE:
-                - quay.io/pypa/manylinux2014_x86_64
-                # Disable i686 builds for now: no binary wheels for cryptography,
-                # source build fails, e.g. https://github.com/inducer/pyopencl/pull/421/checks?check_run_id=1781071632
-                # - quay.io/pypa/manylinux2014_i686
-        steps:
-        -   uses: actions/checkout@v2
-        -   name: "Main Script"
-            env:
-                TWINE_USERNAME: __token__
-                TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-                DOCKER_IMAGE: ${{ matrix.DOCKER_IMAGE }}
-
-            run: |
-                pwd
-                ls -la
-
-                # Only perform upload for tag builds, otherwise unset TWINE_USERNAME to prevent
-                if ! [[ $GITHUB_REF == refs/tags* ]]; then
-                    echo "Not a tag build, GITHUB_REF is '$GITHUB_REF'. Unsetting TWINE_USERNAME"
-                    unset TWINE_USERNAME
-                fi
-
-                if [[ $DOCKER_IMAGE == *i686* ]]; then
-                    PRE_CMD=linux32
-                else
-                    PRE_CMD=""
-                fi
-
-                docker run --rm -v `pwd`:/io -e TWINE_USERNAME -e TWINE_PASSWORD $DOCKER_IMAGE $PRE_CMD /io/scripts/build-wheels.sh
-                ls wheelhouse/
-
     downstream_tests:
         strategy:
             matrix:

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -0,0 +1,65 @@
+name: Build and upload to PyPI
+
+# Build on every branch push, tag push, and pull request change:
+on:
+    push:
+        branches:
+        - main
+        tags:
+        - v*
+    pull_request:
+    schedule:
+        - cron:  '17 3 * * 0'
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, windows-2019, macos-10.15]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: dist/*.tar.gz
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    # upload to PyPI on every tag starting with 'v'
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    # alternatively, to publish when a GitHub Release is created, use the following rule:
+    # if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifact
+          path: dist
+
+      - uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{ secrets.TWINE_PASSWORD }}
+          # To test: repository_url: https://test.pypi.org/legacy/
diff --git a/.gitignore b/.gitignore
@@ -66,3 +66,5 @@ cffi_build.py
 .cache
 .pytest_cache
 .idea
+
+wheelhouse
diff --git a/aksetup_helper.py b/aksetup_helper.py
@@ -1,3 +1,4 @@
+import os
 import sys
 try:
     from setuptools import Extension
@@ -256,7 +257,7 @@ def expand_options(options):
 
 
 class ConfigSchema:
-    def __init__(self, options, conf_file="siteconf.py", conf_dir="."):
+    def __init__(self, options, conf_file="siteconf.py", conf_dir=os.path.dirname(__file__)):
         self.optdict = dict((opt.name, opt) for opt in options)
         self.options = options
         self.conf_dir = conf_dir

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,35 @@
+[build-system]
+# Minimum requirements for the build system to execute.
+requires = ["setuptools>=42.0.0", "wheel>=0.34.2", "Cython", "oldest-supported-numpy", "pybind11>=2.5.0"]  # PEP 508 specifications.
+build-backend = "setuptools.build_meta"
+
+
+[tool.cibuildwheel]
+test-command = "pytest {project}/test"
+test-extras = ["test"]
+
+[tool.cibuildwheel.linux]
+test-command = ""
+before-all = [
+    "yum install -y git openssl-devel ruby",
+    "bash {package}/scripts/build-ocl.sh",
+]
+
+[[tool.cibuildwheel.overrides]]
+select = "*-musllinux*"
+before-all = [
+    "apk add ruby git openssl-dev",
+    "bash {package}/scripts/build-ocl.sh",
+]
+
+[tool.cibuildwheel.macos]
+skip = "pp*"
+before-all = "bash {package}/scripts/build-ocl-macos.sh"
+test-command = "pytest {project}/test/test_array.py" # same limitation as conda-forge
+# https://github.com/conda-forge/pyopencl-feedstock/blob/6f3c5de59b18c9518abba3cb94f6ae92964553f8/recipe/meta.yaml#L62-L63
+
+[tool.cibuildwheel.windows]
+skip = ["*-win32", "pp*"]
+test-command = ""
+before-all = "bash {package}/scripts/build-ocl-windows.sh"
+before-build = "python configure.py --cxxflags=-ID:/a/pyopencl/pyopencl/OpenCL-Headers/install/include --ldflags=\"/LIBPATH:C:/Program Files/OpenCL-ICD-Loader/lib\""
diff --git a/scripts/build-ocl-macos.sh b/scripts/build-ocl-macos.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+set -o xtrace
+
+git clone --branch v2022.01.04 https://github.com/KhronosGroup/OpenCL-ICD-Loader
+git clone --branch v2022.01.04 https://github.com/KhronosGroup/OpenCL-Headers
+
+
+
+cmake -D CMAKE_INSTALL_PREFIX=./OpenCL-Headers/install -S ./OpenCL-Headers -B ./OpenCL-Headers/build 
+cmake --build ./OpenCL-Headers/build --target install
+
+cmake -D CMAKE_PREFIX_PATH=${PWD}/OpenCL-Headers/install -D OPENCL_ICD_LOADER_HEADERS_DIR=${PWD}/OpenCL-Headers/install/include -D CMAKE_INSTALL_PREFIX=./OpenCL-ICD-Loader/install -S ./OpenCL-ICD-Loader -B ./OpenCL-ICD-Loader/build 
+cmake --build ./OpenCL-ICD-Loader/build --target install --config Release
+
+echo "PyOpenCL wheel includes Khronos Group OpenCL-ICD-Loader which is licensed as below" >> ${SCRIPT_DIR}/../LICENSE
+cat ./OpenCL-ICD-Loader/LICENSE >> ${SCRIPT_DIR}/../LICENSE
diff --git a/scripts/build-ocl-windows.sh b/scripts/build-ocl-windows.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+set -o xtrace
+
+git clone --branch v2022.01.04 https://github.com/KhronosGroup/OpenCL-ICD-Loader
+
+git clone --branch v2022.01.04 https://github.com/KhronosGroup/OpenCL-Headers
+
+
+cmake -D CMAKE_INSTALL_PREFIX=./OpenCL-Headers/install -S ./OpenCL-Headers -B ./OpenCL-Headers/build 
+cmake --build ./OpenCL-Headers/build --target install
+
+# if someone would like to try to create win32 wheels bellow lines may be useful
+# cmake -D CMAKE_PREFIX_PATH=${PWD}/OpenCL-Headers/install -DOPENCL_ICD_LOADER_HEADERS_DIR=${PWD}/OpenCL-Headers/install/include -S ./OpenCL-ICD-Loader -B ./OpenCL-ICD-Loader/build 
+# cmake --build ./OpenCL-ICD-Loader/build --target install --config Release
+
+cmake -D CMAKE_PREFIX_PATH=${PWD}/OpenCL-Headers/install -D OPENCL_ICD_LOADER_HEADERS_DIR=${PWD}/OpenCL-Headers/install/include -S ./OpenCL-ICD-Loader -B ./OpenCL-ICD-Loader/build2 -A x64
+cmake --build ./OpenCL-ICD-Loader/build2 --target install --config Release
+
+echo "PyOpenCL wheel includes Khronos Group OpenCL-ICD-Loader which is licensed as below" >> ${SCRIPT_DIR}/../LICENSE
+cat ./OpenCL-ICD-Loader/LICENSE >> ${SCRIPT_DIR}/../LICENSE
diff --git a/scripts/build-ocl.sh b/scripts/build-ocl.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+set -e -x
+
+mkdir -p ~/deps
+cd ~/deps
+
+git clone --branch v2.3.1 https://github.com/OCL-dev/ocl-icd
+cd ocl-icd
+curl -L -O https://raw.githubusercontent.com/conda-forge/ocl-icd-feedstock/e2c03e3ddb1ff86630ccf80dc7b87a81640025ea/recipe/install-headers.patch
+git apply install-headers.patch
+curl -L -O https://github.com/isuruf/ocl-icd/commit/3862386b51930f95d9ad1089f7157a98165d5a6b.patch
+git apply 3862386b51930f95d9ad1089f7157a98165d5a6b.patch
+autoreconf -i
+chmod +x configure
+./configure --prefix=/usr
+make -j4
+make install
+
+# Bundle license files
+echo "PyOpenCL wheel includes ocl-icd which is licensed as below" >> ${SCRIPT_DIR}/../LICENSE
+cat ~/deps/ocl-icd/COPYING >> ${SCRIPT_DIR}/../LICENSE
diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh
diff --git a/setup.py b/setup.py
@@ -31,6 +31,8 @@
 import sys
 from os.path import exists
 
+sys.path.append(os.path.dirname(__file__))
+
 
 def get_config_schema():
     from aksetup_helper import (ConfigSchema, Option,
@@ -161,31 +163,6 @@ def main():
 
     exec(compile(version_file_contents, "pyopencl/version.py", "exec"), ver_dic)
 
-    try:
-        import mako  # noqa
-    except ImportError:
-        print(SEPARATOR)
-        print("Mako is not installed.")
-        print(SEPARATOR)
-        print("That is not a problem, as most of PyOpenCL will be just fine ")
-        print("without it. Some higher-level parts of pyopencl (such as ")
-        print("pyopencl.reduction) will not function without the templating engine ")
-        print("Mako [1] being installed. If you would like this functionality to ")
-        print("work, you might want to install Mako after you finish ")
-        print("installing PyOpenCL.")
-        print("")
-        print("Simply type")
-        print("python -m pip install mako")
-        print("either now or after the installation completes to fix this.")
-        print("")
-        print("[1] http://www.makotemplates.org/")
-        print(SEPARATOR)
-        print("Hit Ctrl-C now if you'd like to think about the situation.")
-        print(SEPARATOR)
-
-        from aksetup_helper import count_down_delay
-        count_down_delay(delay=5)
-
     if not exists("pyopencl/compyte/dtypes.py"):
         print(75 * "-")
         print("You are missing important files from the pyopencl distribution.")
@@ -263,6 +240,7 @@ def main():
             extras_require={
                 "pocl":  ["pocl_binary_distribution>=1.2"],
                 "oclgrind":  ["oclgrind_binary_distribution>=18.3"],
+                "test": ["pytest>=7.0.0", "Mako"],
             },
             include_package_data=True,
             package_data={

diff --git a/test/test_wrapper.py b/test/test_wrapper.py
@@ -637,8 +637,8 @@ def test_vector_args(ctx_factory):
 def test_header_dep_handling(ctx_factory):
     context = ctx_factory()
 
-    from os.path import exists
-    assert exists("empty-header.h")  # if this fails, change dir to pyopencl/test
+    from os.path import exists, dirname, join
+    assert exists(join(dirname(__file__), "empty-header.h"))
 
     kernel_src = """
     #include <empty-header.h>
@@ -648,10 +648,8 @@ def test_header_dep_handling(ctx_factory):
     }
     """
 
-    import os
-
-    cl.Program(context, kernel_src).build(["-I", os.getcwd()])
-    cl.Program(context, kernel_src).build(["-I", os.getcwd()])
+    cl.Program(context, kernel_src).build(["-I", dirname(__file__)])
+    cl.Program(context, kernel_src).build(["-I", dirname(__file__)])
 
 
 def test_context_dep_memoize(ctx_factory):
-Original file line number
+Diff line change
@@ Expand Up / @@ -66,3 +66,5 @@ cffi_build.py @@
     .cache
     .pytest_cache
     .idea
+    wheelhouse