Add a cibuildwheel workflow

arvidn · Nov 28, 2021 · 26f398c · 26f398c
1 parent beb2e63
commit 26f398c
Show file tree

Hide file tree

Showing 9 changed files with 413 additions and 0 deletions.
diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml
@@ -0,0 +1,142 @@
+name: cibuildwheel
+
+# Note: We use a dynamic matrix to build different sets of wheels under
+# different conditions. On workflow_dispatch, we build the full suite of
+# wheels. This takes hours, so on pull_request, we just build a representative
+# sample.
+
+# The full list of cibuildwheel's build targets can be found here:
+# https://github.com/pypa/cibuildwheel/blob/v2.2.0a1/cibuildwheel/resources/build-platforms.toml
+
+# Notes on build targets we (don't) support:
+# - pypy: libtorrent doesn't build with pypy as of writing
+# - macos_arm64: can be cross-compiled from x86_64, but not run, so can't be
+#   tested. Build output indicates it isn't building correctly
+# - macos_universal2: b2 / setup.py doesn't have a straightforward way to build
+#   this as of writing
+# - abi3: Not supported by boost-python (or pybind11) or cibuildwheel as of
+#   writing
+
+on:
+  workflow_dispatch:
+    inputs:
+      publish:
+        description: Write 'PUBLISH' to publish to pypi. BEWARE! ARTIFACTS ARE IMMUTABLE AND CANNOT BE REPLACED ONCE PUBLISHED!
+      publish_test:
+        description: Write 'PUBLISH_TEST' to publish to test-pypi. BEWARE! ARTIFACTS ARE IMMUTABLE AND CANNOT BE REPLACED ONCE PUBLISHED!
+
+  pull_request:
+    paths:
+      - .github/workflows/cibuildwheel.yml
+      - tools/cibuildwheel/**
+      - pyproject.toml
+
+jobs:
+  configure_matrix:
+    runs-on: ubuntu-latest
+    env:
+      # github actions syntax doesn't allow us to have yaml structures as
+      # an input to a job. These environment variables are literal json strings
+      MATRIX_PULL_REQUEST: |
+        {
+          "include": [
+            {"os": "ubuntu-20.04", "CIBW_BUILD": "cp37-manylinux_*", "CIBW_ARCHS": "x86_64"},
+            {"os": "ubuntu-20.04", "CIBW_BUILD": "cp37-musllinux_*", "CIBW_ARCHS": "x86_64"},
+            {"os": "macos-10.15", "CIBW_BUILD": "cp37-*", "CIBW_ARCHS": "x86_64"},
+            {"os": "windows-2019", "CIBW_BUILD": "cp37-*", "CIBW_ARCHS": "AMD64"}
+          ]
+        }
+      MATRIX_WORKFLOW_DISPATCH: |
+        {
+          "include": [
+            {"os": "ubuntu-20.04", "CIBW_BUILD": "cp*-manylinux_*", "CIBW_ARCHS": "x86_64"},
+            {"os": "ubuntu-20.04", "CIBW_BUILD": "cp*-manylinux_*", "CIBW_ARCHS": "aarch64"},
+            {"os": "ubuntu-20.04", "CIBW_BUILD": "cp*-musllinux_*", "CIBW_ARCHS": "x86_64"},
+            {"os": "ubuntu-20.04", "CIBW_BUILD": "cp*-musllinux_*", "CIBW_ARCHS": "aarch64"},
+            {"os": "macos-10.15", "CIBW_BUILD": "cp*", "CIBW_ARCHS": "x86_64"},
+            {"os": "windows-2019", "CIBW_BUILD": "cp*", "CIBW_ARCHS": "x86"},
+            {"os": "windows-2019", "CIBW_BUILD": "cp*", "CIBW_ARCHS": "AMD64"}
+          ]
+        }
+
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+    steps:
+    - id: set-matrix
+      run: |
+        if [ $GITHUB_EVENT_NAME == "pull_request" ]; then
+          echo ::set-output name=matrix::$(echo $MATRIX_PULL_REQUEST | jq -c)
+        else
+          echo ::set-output name=matrix::$(echo $MATRIX_WORKFLOW_DISPATCH | jq -c)
+        fi
+
+  build_wheels:
+    needs: configure_matrix
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.configure_matrix.outputs.matrix) }}
+
+    env:
+      CIBW_BUILD_VERBOSITY: 1
+      CIBW_BUILD: ${{ matrix.CIBW_BUILD }}
+      CIBW_ARCHS: ${{ matrix.CIBW_ARCHS }}
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - uses: actions/cache@v2
+      id: cache-wheel
+      with:
+        path: wheelhouse
+        key: wheel-${{ matrix.CIBW_BUILD }}-${{ matrix.CIBW_ARCHS }}-${{ github.sha }}
+
+    - uses: docker/setup-qemu-action@v1
+      if: steps.cache-wheel.outputs.cache-hit != 'true' && runner.os == 'Linux'
+
+    - uses: pypa/[email protected]
+      if: steps.cache-wheel.outputs.cache-hit != 'true'
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: wheelhouse/*.whl
+        name: wheels
+
+  upload_pypi:
+    needs: build_wheels
+    runs-on: ubuntu-latest
+    if: needs.build_wheels.result == 'success' && github.event.inputs.publish == 'PUBLISH'
+
+    steps:
+    - uses: actions/download-artifact@v2
+      with:
+        name: wheels
+        path: wheelhouse
+
+    - uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        packages_dir: wheelhouse
+        skip_existing: true
+
+  upload_pypi_test:
+    needs: build_wheels
+    runs-on: ubuntu-latest
+    if: needs.build_wheels.result == 'success' && github.event.inputs.publish_test == 'PUBLISH_TEST'
+
+    steps:
+    - uses: actions/download-artifact@v2
+      with:
+        name: wheels
+        path: wheelhouse
+
+    - uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        packages_dir: wheelhouse
+        skip_existing: true
+        repository_url: https://test.pypi.org/legacy/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -26,6 +26,11 @@ repos:
     -   id: debug-statements
     -   id: check-symlinks
     -   id: check-toml
+- repo: https://github.com/pappasam/toml-sort
+  rev: v0.19.0
+  hooks:
+  - id: toml-sort
+    args: [--all, --in-place]
 - repo: https://github.com/pre-commit/pygrep-hooks
   rev: v1.7.0
   hooks:

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,3 +1,88 @@
+[project]
+requires-python = ">=3.7"
+
+[tool.cibuildwheel]
+skip = "pp*"
+
+[tool.cibuildwheel.macos]
+before-all = [
+    "./tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT",
+    "brew install openssl",
+]
+test-command = [
+    "cd {project}/bindings/python",
+    "python test.py",
+]
+
+[tool.cibuildwheel.macos.environment]
+BOOST_BUILD_PATH = "/tmp/boost/tools/build"
+BOOST_ROOT = "/tmp/boost"
+BOOST_VERSION = "1.76.0"
+MACOSX_DEPLOYMENT_TARGET = "10.9"  # required for full C++11 support
+PATH = "/tmp/boost:$PATH"
+
+[[tool.cibuildwheel.overrides]]
+before-all = [
+    "./tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT",
+    "yum install -y glibc-static",  # needed for libutil.a and libdl.a
+    "./tools/cibuildwheel/setup_ccache_on_manylinux.sh",
+    "./tools/cibuildwheel/setup_openssl.sh",
+]
+before-test = "ccache -s"
+select = "*-manylinux_*"
+test-command = [
+    "cd {project}/bindings/python",
+    "python test.py",
+]
+
+[tool.cibuildwheel.overrides.environment]  # sub-table of previous block!
+BOOST_BUILD_PATH = "/tmp/boost/tools/build"
+BOOST_ROOT = "/tmp/boost"
+BOOST_VERSION = "1.76.0"
+PATH = "/usr/local/ccache/bin:/tmp/boost:$PATH"
+
+[[tool.cibuildwheel.overrides]]
+before-all = [
+    "./tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT",
+    "apk add ccache openssl-dev openssl-libs-static",
+    "./tools/cibuildwheel/setup_openssl.sh",
+]
+before-test = "ccache -s"
+select = "*-musllinux_*"
+test-command = [
+    "cd {project}/bindings/python",
+    "python test.py",
+]
+
+[tool.cibuildwheel.overrides.environment]  # sub-table of previous block!
+BOOST_BUILD_PATH = "/tmp/boost/tools/build"
+BOOST_ROOT = "/tmp/boost"
+BOOST_VERSION = "1.76.0"
+PATH = "/usr/lib/ccache/bin:/tmp/boost:$PATH"
+
+[[tool.cibuildwheel.overrides]]
+before-all = [
+    "bash -c './tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT'",
+    "bash -c 'choco install --no-progress --x86 openssl'",  # choco only allows EITHER 32 OR 64-bit version of a package
+]
+select = "*-win32"
+
+[[tool.cibuildwheel.overrides]]
+before-all = [
+    "bash -c './tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT'",
+    "bash -c 'choco install --no-progress openssl'",        # choco only allows EITHER 32 OR 64-bit version of a package
+]
+select = "*-win_amd64"
+
+[tool.cibuildwheel.windows]
+test-command = '''bash -c "cd '{project}/bindings/python' && python test.py"'''
+
+[tool.cibuildwheel.windows.environment]
+BOOST_BUILD_PATH = 'c:/boost/tools/build'
+BOOST_ROOT = 'c:/boost'
+BOOST_VERSION = "1.76.0"
+PATH = 'c:/boost;$PATH'
+
 [tool.isort]
 profile = "google"
 single_line_exclusions = []

diff --git a/tools/cibuildwheel/manylinux/build-openssl.sh b/tools/cibuildwheel/manylinux/build-openssl.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Top-level build script called from Dockerfile
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# Get build utilities
+source $MY_DIR/build_utils.sh
+
+# Install a more recent openssl
+check_var ${OPENSSL_ROOT}
+check_var ${OPENSSL_HASH}
+check_var ${OPENSSL_DOWNLOAD_URL}
+
+OPENSSL_VERSION=${OPENSSL_ROOT#*-}
+OPENSSL_MIN_VERSION=1.1.1
+
+INSTALLED=$(openssl version | head -1 | awk '{ print $2 }')
+SMALLEST=$(echo -e "${INSTALLED}\n${OPENSSL_MIN_VERSION}" | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | head -1)
+if [ "${SMALLEST}" == "${OPENSSL_MIN_VERSION}" ]; then
+	echo "skipping installation of openssl ${OPENSSL_VERSION}, system provides openssl ${INSTALLED} which is newer than openssl ${OPENSSL_MIN_VERSION}"
+	exit 0
+fi
+
+if which yum; then
+	yum erase -y openssl-devel
+else
+	apt-get remove -y libssl-dev
+fi
+
+fetch_source ${OPENSSL_ROOT}.tar.gz ${OPENSSL_DOWNLOAD_URL}
+check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
+tar -xzf ${OPENSSL_ROOT}.tar.gz
+pushd ${OPENSSL_ROOT}
+./config no-shared --prefix=/usr/local/ssl --openssldir=/usr/local/ssl CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS} -fPIC" CXXFLAGS="${MANYLINUX_CXXFLAGS} -fPIC" LDFLAGS="${MANYLINUX_LDFLAGS} -fPIC" > /dev/null
+make > /dev/null
+make install_sw > /dev/null
+popd
+rm -rf ${OPENSSL_ROOT} ${OPENSSL_ROOT}.tar.gz
+
+
+/usr/local/ssl/bin/openssl version
diff --git a/tools/cibuildwheel/manylinux/build_utils.sh b/tools/cibuildwheel/manylinux/build_utils.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Helper utilities for build
+
+
+# use all flags used by ubuntu 20.04 for hardening builds, dpkg-buildflags --export
+# other flags mentioned in https://wiki.ubuntu.com/ToolChain/CompilerFlags can't be
+# used because the distros used here are too old
+MANYLINUX_CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
+MANYLINUX_CFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security"
+MANYLINUX_CXXFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security"
+MANYLINUX_LDFLAGS="-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now"
+
+
+function check_var {
+    if [ -z "$1" ]; then
+        echo "required variable not defined"
+        exit 1
+    fi
+}
+
+
+function fetch_source {
+    # This is called both inside and outside the build context (e.g. in Travis) to prefetch
+    # source tarballs, where curl exists (and works)
+    local file=$1
+    check_var ${file}
+    local url=$2
+    check_var ${url}
+    if [ -f ${file} ]; then
+        echo "${file} exists, skipping fetch"
+    else
+        curl -fsSL -o ${file} ${url}/${file}
+    fi
+}
+
+
+function check_sha256sum {
+    local fname=$1
+    check_var ${fname}
+    local sha256=$2
+    check_var ${sha256}
+
+    echo "${sha256}  ${fname}" > ${fname}.sha256
+    sha256sum -c ${fname}.sha256
+    rm -f ${fname}.sha256
+}
+
+
+function do_standard_install {
+    ./configure "$@" CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" "CXXFLAGS=${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}" > /dev/null
+    make > /dev/null
+    make install > /dev/null
+}
+
+function strip_ {
+	# Strip what we can -- and ignore errors, because this just attempts to strip
+	# *everything*, including non-ELF files:
+	find $1 -type f -print0 | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true
+}
+
+function clean_pyc {
+	find $1 -type f -a \( -name '*.pyc' -o -name '*.pyo' \) -delete
+}
diff --git a/tools/cibuildwheel/manylinux/openssl-version.sh b/tools/cibuildwheel/manylinux/openssl-version.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export OPENSSL_ROOT=openssl-1.1.1l
+export OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
+export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source
diff --git a/tools/cibuildwheel/setup_boost.sh b/tools/cibuildwheel/setup_boost.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# This script is meant to be called by cibuildwheel. It should run on github
+# actions Linux, Mac and Windows.
+
+set -ex
+
+VERSION="$1"
+BOOST_ROOT="$2"
+
+VUNDER="${VERSION//./_}"
+
+TEMP_ARCHIVE="$(mktemp)"
+
+mkdir -p "$BOOST_ROOT"
+
+curl -L -o "$TEMP_ARCHIVE" "https://boostorg.jfrog.io/artifactory/main/release/${VERSION}/source/boost_${VUNDER}.tar.gz"
+
+tar -z -x -C "$BOOST_ROOT" -f "$TEMP_ARCHIVE" --strip-components 1
+rm "$TEMP_ARCHIVE"
+
+cd "$BOOST_ROOT"
+
+./bootstrap.sh
+
+cat ./project-config.jam
+
+./b2 headers