From e8751bf7abc2b115165671571437c360fcd4955a Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Sat, 4 Sep 2021 19:10:35 +0800 Subject: [PATCH] Build python wheels and install python package using pip --- .travis.yml | 38 ++++++++++++++---------- appveyor.yml | 1 + build_manylinux_wheel.sh | 38 ++++++++++++++++++++++++ docker/Dockerfile.manylinux2010 | 42 +++++++++++++++++++++++++++ python/CMakeLists.txt | 32 +++++++++++++++++---- python/__init__.py | 4 +++ python/_version.py | 3 ++ python/setup.py | 51 +++++++++++++++++++++++++++++++++ 8 files changed, 188 insertions(+), 21 deletions(-) create mode 100755 build_manylinux_wheel.sh create mode 100644 docker/Dockerfile.manylinux2010 create mode 100644 python/__init__.py create mode 100644 python/_version.py create mode 100644 python/setup.py diff --git a/.travis.yml b/.travis.yml index 184fc16..85f6807 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,23 +7,31 @@ jobs: dist: xenial - os: osx osx_image: xcode11 + - sudo: required + services: + - docker + env: DOCKER_IMAGE=kontinuation/fmm-manylinux:latest before_install: - - chmod +x travis_install_dependency.sh + - if [ -z "$DOCKER_IMAGE" ]; then chmod +x travis_install_dependency.sh; fi install: - - ./travis_install_dependency.sh -script: - - if [ "$TRAVIS_OS_NAME" = "osx" ]; then export PATH=/usr/bin:${PATH}; fi - - mkdir -p build - - cd build - - cmake .. - - make -j 4 - - sudo make install - - fmm - - stmatch - - ubodt_gen - - cd ../example/python - - which python - - python fmm_test.py + - if [ -z "$DOCKER_IMAGE" ]; then ./travis_install_dependency.sh; else docker pull $DOCKER_IMAGE; fi +script: | + if [ -n "$DOCKER_IMAGE" ]; then + docker run --rm -v `pwd`:/repo $DOCKER_IMAGE /repo/build_manylinux_wheel.sh; + else + if [ "$TRAVIS_OS_NAME" = "osx" ]; then export PATH=/usr/bin:${PATH}; fi + mkdir -p build + cd build + cmake .. + make -j 4 + sudo make install + fmm + stmatch + ubodt_gen + cd ../example/python + which python + python fmm_test.py + fi branches: only: - master diff --git a/appveyor.yml b/appveyor.yml index 3793707..ddff7d2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,6 +19,7 @@ init: - '%CYG_SH% "install apt-cyg /bin"' - '%CYG_SH% "apt-cyg install make gcc-g++ cmake gdal libboost-devel libgdal-devel"' - '%CYG_SH% "apt-cyg install swig python-devel"' + - '%CYG_SH% "python -m pip install setuptools wheel"' # clone directory clone_folder: C:/cygwin64/home/appveyor/fmm diff --git a/build_manylinux_wheel.sh b/build_manylinux_wheel.sh new file mode 100755 index 0000000..4e9512d --- /dev/null +++ b/build_manylinux_wheel.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Note: this script should run in manylinux2010 docker container. Please +# use docker/Dockerfile.manylinux2010 to build the image. + +set -e + +script_file=$(readlink -f $0) +script_dir=$(dirname $script_file) +cd "$script_dir" + +version=$1 +if [ -n "$version" ]; then + echo "Rewriting python/_version.py" + sed -i s"/^__version__\s*=.*/__version__ = '$version'/" python/_version.py +fi + +echo "Building FMM package for cp38-cp38" +export PATH=/opt/python/cp38-cp38/bin:$PATH +mkdir -p build && cd build +cmake .. -DPYTHON_INCLUDE_DIR=$(python -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") -DPYTHON_LIBRARY=$(python -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))") +make -j2 +make install +auditwheel repair python/dist/*.whl + +echo "Building FMM package for cp37-cp37m" +export PATH=/opt/python/cp37-cp37m/bin:$PATH +rm -rf python +cmake .. -DPYTHON_INCLUDE_DIR=$(python -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") -DPYTHON_LIBRARY=$(python -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))") +make +auditwheel repair python/dist/*.whl + +echo "Building FMM package for cp36-cp36m" +export PATH=/opt/python/cp36-cp36m/bin:$PATH +rm -rf python +cmake .. -DPYTHON_INCLUDE_DIR=$(python -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") -DPYTHON_LIBRARY=$(python -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))") +make +auditwheel repair python/dist/*.whl diff --git a/docker/Dockerfile.manylinux2010 b/docker/Dockerfile.manylinux2010 new file mode 100644 index 0000000..62ad083 --- /dev/null +++ b/docker/Dockerfile.manylinux2010 @@ -0,0 +1,42 @@ +FROM quay.io/pypa/manylinux2010_x86_64:latest + +RUN yum install -y wget curl-devel zlib-devel libzstd-devel libwebp-devel hdf5-devel expat-devel + +# Install sqlite3 +RUN mkdir -p /build/sqlite3 && \ + cd /build/sqlite3 && \ + wget https://www.sqlite.org/2021/sqlite-autoconf-3360000.tar.gz && \ + tar xf sqlite-autoconf-3360000.tar.gz && \ + cd sqlite-autoconf-3360000 && \ + CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA=1" ./configure --prefix=/usr/local && \ + make && make install && ldconfig + +# Install boost +RUN mkdir -p /build/boost && \ + cd /build/boost && \ + wget https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \ + tar xf boost_1_71_0.tar.bz2 && \ + cd boost_1_71_0 && \ + ./bootstrap.sh --prefix=/usr/local && \ + ./b2 install --prefix=/usr/local --with=all || true && \ + ldconfig # some modules may fail to install, but it does not matter + +# Install proj6 +RUN mkdir -p /build/proj6 && \ + cd /build/proj6 && \ + wget https://download.osgeo.org/proj/proj-6.1.1.tar.gz && \ + tar xf proj-6.1.1.tar.gz && \ + cd proj-6.1.1 && \ + ./configure --prefix=/usr/local && \ + make -j4 && make install && ldconfig + +# Install gdal +RUN mkdir -p /build/gdal && \ + cd /build/gdal && \ + wget https://download.osgeo.org/gdal/2.4.4/gdal-2.4.4.tar.gz && \ + tar xf gdal-2.4.4.tar.gz && \ + cd gdal-2.4.4 && \ + ./configure --prefix=/usr/local && \ + make -j4 && make install && \ + strip /usr/local/lib/libgdal.so.20 && ldconfig && \ + cd / && rm -rf /build diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 7c042f2..ca1a3d0 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -18,9 +18,15 @@ else() endif() execute_process( - COMMAND python -c "from distutils.sysconfig import get_python_lib;print(get_python_lib())" - OUTPUT_VARIABLE PYTHON_SITE_PACKAGES OUTPUT_STRIP_TRAILING_WHITESPACE) -message(STATUS "Python packages ${PYTHON_SITE_PACKAGES}") + COMMAND python -c "from setup import get_wheel_name;print(get_wheel_name())" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE WHEEL_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) + +set(PYTHON_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/setup.py + ${CMAKE_CURRENT_SOURCE_DIR}/_version.py + ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py) +file(COPY ${PYTHON_SOURCES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) set_source_files_properties( ${PROJECT_SOURCE_DIR}/python/fmm.i PROPERTIES CPLUSPLUS ON) @@ -29,7 +35,14 @@ if (${CMAKE_VERSION} VERSION_LESS 3.13.0) message(STATUS "Using swig add module") SWIG_ADD_MODULE(fmm python ${PROJECT_SOURCE_DIR}/python/fmm.i) swig_link_libraries(fmm FMMLIB ${PYTHON_LIBRARIES}) - install(TARGETS ${SWIG_MODULE_fmm_REAL_NAME} DESTINATION ${PYTHON_SITE_PACKAGES}) + add_custom_target(${WHEEL_NAME} ALL + DEPENDS ${SWIG_MODULE_fmm_REAL_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/setup.py + ${CMAKE_CURRENT_BINARY_DIR}/_version.py + ${CMAKE_CURRENT_BINARY_DIR}/__init__.py + COMMAND python setup.py bdist_wheel + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building wheel") else() message(STATUS "Using swig add library") SWIG_ADD_LIBRARY(pyfmm @@ -37,7 +50,14 @@ else() SOURCES ${PROJECT_SOURCE_DIR}/python/fmm.i) set_property(TARGET pyfmm PROPERTY OUTPUT_NAME fmm) swig_link_libraries(pyfmm FMMLIB ${PYTHON_LIBRARIES}) - install(TARGETS pyfmm DESTINATION ${PYTHON_SITE_PACKAGES}) + add_custom_target(${WHEEL_NAME} ALL + DEPENDS pyfmm + ${CMAKE_CURRENT_BINARY_DIR}/setup.py + ${CMAKE_CURRENT_BINARY_DIR}/_version.py + ${CMAKE_CURRENT_BINARY_DIR}/__init__.py + COMMAND python setup.py bdist_wheel + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building wheel") endif() -install(FILES ${CMAKE_BINARY_DIR}/python/fmm.py DESTINATION ${PYTHON_SITE_PACKAGES}) +install(CODE "execute_process(COMMAND python -m pip install --force-reinstall ${CMAKE_CURRENT_BINARY_DIR}/dist/${WHEEL_NAME})") diff --git a/python/__init__.py b/python/__init__.py new file mode 100644 index 0000000..5c650e8 --- /dev/null +++ b/python/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python + +from .fmm import * +from ._version import __version__ diff --git a/python/_version.py b/python/_version.py new file mode 100644 index 0000000..5dc7c53 --- /dev/null +++ b/python/_version.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +__version__ = '0.1.0.dev0' diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 0000000..0ae285d --- /dev/null +++ b/python/setup.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +from setuptools import setup +from setuptools.dist import Distribution +from setuptools.command.install import install +from wheel.bdist_wheel import bdist_wheel as _bdist_wheel +from _version import __version__ + + +# Force building a non-pure (py3-none-any) wheel when extensions were not +# specified in setup configs. What we want is a wheel with platform-specific +# tag such as cp38-cp38-linux_x86_64. +class bdist_wheel(_bdist_wheel): + def finalize_options(self): + _bdist_wheel.finalize_options(self) + self.root_is_pure = False + + +# Workaround a distutils bug, please refer to https://github.com/google/or-tools/issues/616 +class InstallPlatlib(install): + def finalize_options(self): + install.finalize_options(self) + self.install_lib = self.install_platlib + + +def wheel_name(**kwargs): + dist = Distribution(attrs=kwargs) + bdist_wheel_cmd = dist.get_command_obj('bdist_wheel') + bdist_wheel_cmd.ensure_finalized() + distname = bdist_wheel_cmd.wheel_dist_name + tag = '-'.join(bdist_wheel_cmd.get_tag()) + return '%s-%s.whl' % (distname, tag) + + +def get_wheel_name(): + return wheel_name(**setup_args) + + +setup_args = dict( + name='FastMM', + version=__version__, + description='Fast map matching, an open source framework in C++', + url='https://github.com/cyang-kth/fmm', + packages=['fmm'], + package_dir={'fmm': '.'}, + package_data={'fmm': ['*.so', '*.dll', '*.dylib']}, + cmdclass={'bdist_wheel': bdist_wheel, 'install': InstallPlatlib}) + + +if __name__ == '__main__': + setup(**setup_args)