diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml new file mode 100644 index 00000000000..a549c2548e0 --- /dev/null +++ b/.github/workflows/cibuildwheel.yml @@ -0,0 +1,142 @@ +name: cibuildwheel + +# Note: We use a dynamic matrix to build different sets of wheels under +# different conditions. On workflow_dispatch, we build the full suite of +# wheels. This takes hours, so on pull_request, we just build a representative +# sample. + +# The full list of cibuildwheel's build targets can be found here: +# https://github.com/pypa/cibuildwheel/blob/v2.2.0a1/cibuildwheel/resources/build-platforms.toml + +# Notes on build targets we (don't) support: +# - pypy: libtorrent doesn't build with pypy as of writing +# - macos_arm64: can be cross-compiled from x86_64, but not run, so can't be +# tested. Build output indicates it isn't building correctly +# - macos_universal2: b2 / setup.py doesn't have a straightforward way to build +# this as of writing +# - abi3: Not supported by boost-python (or pybind11) or cibuildwheel as of +# writing + +on: + workflow_dispatch: + inputs: + publish: + description: Write 'PUBLISH' to publish to pypi. BEWARE! ARTIFACTS ARE IMMUTABLE AND CANNOT BE REPLACED ONCE PUBLISHED! + publish_test: + description: Write 'PUBLISH_TEST' to publish to test-pypi. BEWARE! ARTIFACTS ARE IMMUTABLE AND CANNOT BE REPLACED ONCE PUBLISHED! + + pull_request: + paths: + - .github/workflows/cibuildwheel.yml + - tools/cibuildwheel/** + - pyproject.toml + +jobs: + configure_matrix: + runs-on: ubuntu-latest + env: + # github actions syntax doesn't allow us to have yaml structures as + # an input to a job. These environment variables are literal json strings + MATRIX_PULL_REQUEST: | + { + "include": [ + {"os": "ubuntu-20.04", "CIBW_BUILD": "cp37-manylinux_*", "CIBW_ARCHS": "x86_64"}, + {"os": "ubuntu-20.04", "CIBW_BUILD": "cp37-musllinux_*", "CIBW_ARCHS": "x86_64"}, + {"os": "macos-10.15", "CIBW_BUILD": "cp37-*", "CIBW_ARCHS": "x86_64"}, + {"os": "windows-2019", "CIBW_BUILD": "cp37-*", "CIBW_ARCHS": "AMD64"} + ] + } + MATRIX_WORKFLOW_DISPATCH: | + { + "include": [ + {"os": "ubuntu-20.04", "CIBW_BUILD": "cp*-manylinux_*", "CIBW_ARCHS": "x86_64"}, + {"os": "ubuntu-20.04", "CIBW_BUILD": "cp*-manylinux_*", "CIBW_ARCHS": "aarch64"}, + {"os": "ubuntu-20.04", "CIBW_BUILD": "cp*-musllinux_*", "CIBW_ARCHS": "x86_64"}, + {"os": "ubuntu-20.04", "CIBW_BUILD": "cp*-musllinux_*", "CIBW_ARCHS": "aarch64"}, + {"os": "macos-10.15", "CIBW_BUILD": "cp*", "CIBW_ARCHS": "x86_64"}, + {"os": "windows-2019", "CIBW_BUILD": "cp*", "CIBW_ARCHS": "x86"}, + {"os": "windows-2019", "CIBW_BUILD": "cp*", "CIBW_ARCHS": "AMD64"} + ] + } + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + steps: + - id: set-matrix + run: | + if [ $GITHUB_EVENT_NAME == "pull_request" ]; then + echo ::set-output name=matrix::$(echo $MATRIX_PULL_REQUEST | jq -c) + else + echo ::set-output name=matrix::$(echo $MATRIX_WORKFLOW_DISPATCH | jq -c) + fi + + build_wheels: + needs: configure_matrix + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.configure_matrix.outputs.matrix) }} + + env: + CIBW_BUILD_VERBOSITY: 1 + CIBW_BUILD: ${{ matrix.CIBW_BUILD }} + CIBW_ARCHS: ${{ matrix.CIBW_ARCHS }} + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - uses: actions/cache@v2 + id: cache-wheel + with: + path: wheelhouse + key: wheel-${{ matrix.CIBW_BUILD }}-${{ matrix.CIBW_ARCHS }}-${{ github.sha }} + + - uses: docker/setup-qemu-action@v1 + if: steps.cache-wheel.outputs.cache-hit != 'true' && runner.os == 'Linux' + + - uses: pypa/cibuildwheel@v2.2.2 + if: steps.cache-wheel.outputs.cache-hit != 'true' + + - uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl + name: wheels + + upload_pypi: + needs: build_wheels + runs-on: ubuntu-latest + if: needs.build_wheels.result == 'success' && github.event.inputs.publish == 'PUBLISH' + + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + path: wheelhouse + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + packages_dir: wheelhouse + skip_existing: true + + upload_pypi_test: + needs: build_wheels + runs-on: ubuntu-latest + if: needs.build_wheels.result == 'success' && github.event.inputs.publish_test == 'PUBLISH_TEST' + + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + path: wheelhouse + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + packages_dir: wheelhouse + skip_existing: true + repository_url: https://test.pypi.org/legacy/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a1f3e4b9c7..17101839c58 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,11 @@ repos: - id: debug-statements - id: check-symlinks - id: check-toml +- repo: https://github.com/pappasam/toml-sort + rev: v0.19.0 + hooks: + - id: toml-sort + args: [--all, --in-place] - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.7.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index eb0366f1e24..517b029cb35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,88 @@ +[project] +requires-python = ">=3.7" + +[tool.cibuildwheel] +skip = "pp*" + +[tool.cibuildwheel.macos] +before-all = [ + "./tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT", + "brew install openssl", +] +test-command = [ + "cd {project}/bindings/python", + "python test.py", +] + +[tool.cibuildwheel.macos.environment] +BOOST_BUILD_PATH = "/tmp/boost/tools/build" +BOOST_ROOT = "/tmp/boost" +BOOST_VERSION = "1.76.0" +MACOSX_DEPLOYMENT_TARGET = "10.9" # required for full C++11 support +PATH = "/tmp/boost:$PATH" + +[[tool.cibuildwheel.overrides]] +before-all = [ + "./tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT", + "yum install -y glibc-static", # needed for libutil.a and libdl.a + "./tools/cibuildwheel/setup_ccache_on_manylinux.sh", + "./tools/cibuildwheel/setup_openssl.sh", +] +before-test = "ccache -s" +select = "*-manylinux_*" +test-command = [ + "cd {project}/bindings/python", + "python test.py", +] + +[tool.cibuildwheel.overrides.environment] # sub-table of previous block! +BOOST_BUILD_PATH = "/tmp/boost/tools/build" +BOOST_ROOT = "/tmp/boost" +BOOST_VERSION = "1.76.0" +PATH = "/usr/local/ccache/bin:/tmp/boost:$PATH" + +[[tool.cibuildwheel.overrides]] +before-all = [ + "./tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT", + "apk add ccache openssl-dev openssl-libs-static", + "./tools/cibuildwheel/setup_openssl.sh", +] +before-test = "ccache -s" +select = "*-musllinux_*" +test-command = [ + "cd {project}/bindings/python", + "python test.py", +] + +[tool.cibuildwheel.overrides.environment] # sub-table of previous block! +BOOST_BUILD_PATH = "/tmp/boost/tools/build" +BOOST_ROOT = "/tmp/boost" +BOOST_VERSION = "1.76.0" +PATH = "/usr/lib/ccache/bin:/tmp/boost:$PATH" + +[[tool.cibuildwheel.overrides]] +before-all = [ + "bash -c './tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT'", + "bash -c 'choco install --no-progress --x86 openssl'", # choco only allows EITHER 32 OR 64-bit version of a package +] +select = "*-win32" + +[[tool.cibuildwheel.overrides]] +before-all = [ + "bash -c './tools/cibuildwheel/setup_boost.sh $BOOST_VERSION $BOOST_ROOT'", + "bash -c 'choco install --no-progress openssl'", # choco only allows EITHER 32 OR 64-bit version of a package +] +select = "*-win_amd64" + +[tool.cibuildwheel.windows] +test-command = '''bash -c "cd '{project}/bindings/python' && python test.py"''' + +[tool.cibuildwheel.windows.environment] +BOOST_BUILD_PATH = 'c:/boost/tools/build' +BOOST_ROOT = 'c:/boost' +BOOST_VERSION = "1.76.0" +PATH = 'c:/boost;$PATH' + [tool.isort] profile = "google" single_line_exclusions = [] diff --git a/tools/cibuildwheel/manylinux/build-openssl.sh b/tools/cibuildwheel/manylinux/build-openssl.sh new file mode 100755 index 00000000000..e39fbf0ab02 --- /dev/null +++ b/tools/cibuildwheel/manylinux/build-openssl.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install a more recent openssl +check_var ${OPENSSL_ROOT} +check_var ${OPENSSL_HASH} +check_var ${OPENSSL_DOWNLOAD_URL} + +OPENSSL_VERSION=${OPENSSL_ROOT#*-} +OPENSSL_MIN_VERSION=1.1.1 + +INSTALLED=$(openssl version | head -1 | awk '{ print $2 }') +SMALLEST=$(echo -e "${INSTALLED}\n${OPENSSL_MIN_VERSION}" | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | head -1) +if [ "${SMALLEST}" == "${OPENSSL_MIN_VERSION}" ]; then + echo "skipping installation of openssl ${OPENSSL_VERSION}, system provides openssl ${INSTALLED} which is newer than openssl ${OPENSSL_MIN_VERSION}" + exit 0 +fi + +if which yum; then + yum erase -y openssl-devel +else + apt-get remove -y libssl-dev +fi + +fetch_source ${OPENSSL_ROOT}.tar.gz ${OPENSSL_DOWNLOAD_URL} +check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH} +tar -xzf ${OPENSSL_ROOT}.tar.gz +pushd ${OPENSSL_ROOT} +./config no-shared --prefix=/usr/local/ssl --openssldir=/usr/local/ssl CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS} -fPIC" CXXFLAGS="${MANYLINUX_CXXFLAGS} -fPIC" LDFLAGS="${MANYLINUX_LDFLAGS} -fPIC" > /dev/null +make > /dev/null +make install_sw > /dev/null +popd +rm -rf ${OPENSSL_ROOT} ${OPENSSL_ROOT}.tar.gz + + +/usr/local/ssl/bin/openssl version diff --git a/tools/cibuildwheel/manylinux/build_utils.sh b/tools/cibuildwheel/manylinux/build_utils.sh new file mode 100755 index 00000000000..45f59316601 --- /dev/null +++ b/tools/cibuildwheel/manylinux/build_utils.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Helper utilities for build + + +# use all flags used by ubuntu 20.04 for hardening builds, dpkg-buildflags --export +# other flags mentioned in https://wiki.ubuntu.com/ToolChain/CompilerFlags can't be +# used because the distros used here are too old +MANYLINUX_CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" +MANYLINUX_CFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security" +MANYLINUX_CXXFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security" +MANYLINUX_LDFLAGS="-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now" + + +function check_var { + if [ -z "$1" ]; then + echo "required variable not defined" + exit 1 + fi +} + + +function fetch_source { + # This is called both inside and outside the build context (e.g. in Travis) to prefetch + # source tarballs, where curl exists (and works) + local file=$1 + check_var ${file} + local url=$2 + check_var ${url} + if [ -f ${file} ]; then + echo "${file} exists, skipping fetch" + else + curl -fsSL -o ${file} ${url}/${file} + fi +} + + +function check_sha256sum { + local fname=$1 + check_var ${fname} + local sha256=$2 + check_var ${sha256} + + echo "${sha256} ${fname}" > ${fname}.sha256 + sha256sum -c ${fname}.sha256 + rm -f ${fname}.sha256 +} + + +function do_standard_install { + ./configure "$@" CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" "CXXFLAGS=${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}" > /dev/null + make > /dev/null + make install > /dev/null +} + +function strip_ { + # Strip what we can -- and ignore errors, because this just attempts to strip + # *everything*, including non-ELF files: + find $1 -type f -print0 | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true +} + +function clean_pyc { + find $1 -type f -a \( -name '*.pyc' -o -name '*.pyo' \) -delete +} diff --git a/tools/cibuildwheel/manylinux/openssl-version.sh b/tools/cibuildwheel/manylinux/openssl-version.sh new file mode 100755 index 00000000000..8c645e1bad8 --- /dev/null +++ b/tools/cibuildwheel/manylinux/openssl-version.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +export OPENSSL_ROOT=openssl-1.1.1l +export OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 +export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source diff --git a/tools/cibuildwheel/setup_boost.sh b/tools/cibuildwheel/setup_boost.sh new file mode 100755 index 00000000000..c369d01050b --- /dev/null +++ b/tools/cibuildwheel/setup_boost.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# This script is meant to be called by cibuildwheel. It should run on github +# actions Linux, Mac and Windows. + +set -ex + +VERSION="$1" +BOOST_ROOT="$2" + +VUNDER="${VERSION//./_}" + +TEMP_ARCHIVE="$(mktemp)" + +mkdir -p "$BOOST_ROOT" + +curl -L -o "$TEMP_ARCHIVE" "https://boostorg.jfrog.io/artifactory/main/release/${VERSION}/source/boost_${VUNDER}.tar.gz" + +tar -z -x -C "$BOOST_ROOT" -f "$TEMP_ARCHIVE" --strip-components 1 +rm "$TEMP_ARCHIVE" + +cd "$BOOST_ROOT" + +./bootstrap.sh + +cat ./project-config.jam + +./b2 headers diff --git a/tools/cibuildwheel/setup_ccache_on_manylinux.sh b/tools/cibuildwheel/setup_ccache_on_manylinux.sh new file mode 100755 index 00000000000..e7f9e4fa80f --- /dev/null +++ b/tools/cibuildwheel/setup_ccache_on_manylinux.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -xe + +if [ $(uname -m) != x86_64 -a $(uname -m) != aarch64 ] +then + echo "ccache isn't known to exist on $(uname -m). skipping ccache setup" + exit 0 +fi + +yum install -y epel-release # overlay containing ccache +yum install -y ccache + +# The symlinks in /usr/lib64/ccache are auto-managed by rpm postinstall +# scripts. They are only created if appropriate packages are installed. However +# this management only knows about the standard gcc* packages, not the +# devtoolset packages, so they don't get created correctly on manylinux. We try +# to create them ourselves. +mkdir -p /usr/local/ccache/bin +for path in /opt/rh/devtoolset-*/root/usr/bin/*cc /opt/rh/devtoolset-*/root/usr/bin/*cc-[0-9]* /opt/rh/devtoolset-*/root/usr/bin/*++ /opt/rh/devtoolset-*/root/usr/bin/*++-[0-9]* /opt/rh/devtoolset-*/root/usr/bin/*cpp /opt/rh/devtoolset-*/root/usr/bin/*cpp-[0-9]* +do + ln -s /usr/bin/ccache "/usr/local/ccache/bin/$(basename "$path")" +done diff --git a/tools/cibuildwheel/setup_openssl.sh b/tools/cibuildwheel/setup_openssl.sh new file mode 100755 index 00000000000..dcd19d85058 --- /dev/null +++ b/tools/cibuildwheel/setup_openssl.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -xe + +TOOLS=$(dirname "$(readlink -f "$0")") + +. "$TOOLS/manylinux/openssl-version.sh" +manylinux-entrypoint "$TOOLS/manylinux/build-openssl.sh" + +# If the build script finds a new enough openssl on the system, it will skip building. + +if [ -d /usr/local/ssl ] +then + ln -s /usr/local/ssl/include/openssl /usr/local/include + ln -s /usr/local/ssl/lib/libcrypto.a /usr/local/lib + ln -s /usr/local/ssl/lib/libssl.a /usr/local/lib +fi