Build python wheels and install python package using pip

.travis.yml

@@ -7,23 +7,31 @@ jobs:
       dist: xenial
     - os: osx
       osx_image: xcode11
+    - sudo: required
+      services:
+        - docker
+      env: DOCKER_IMAGE=kontinuation/fmm-manylinux:latest
 before_install:
-  - chmod +x travis_install_dependency.sh
+  - if [ -z "$DOCKER_IMAGE" ]; then chmod +x travis_install_dependency.sh; fi
 install:
-  - ./travis_install_dependency.sh
-script:
-  - if [ "$TRAVIS_OS_NAME" = "osx" ]; then export PATH=/usr/bin:${PATH}; fi
-  - mkdir -p build
-  - cd build
-  - cmake ..
-  - make -j 4
-  - sudo make install
-  - fmm
-  - stmatch
-  - ubodt_gen
-  - cd ../example/python
-  - which python
-  - python fmm_test.py
+  - if [ -z "$DOCKER_IMAGE" ]; then ./travis_install_dependency.sh; else docker pull $DOCKER_IMAGE; fi
+script: |
+  if [ -n "$DOCKER_IMAGE" ]; then
+    docker run --rm -v `pwd`:/repo $DOCKER_IMAGE /repo/build_manylinux_wheel.sh;
+  else
+    if [ "$TRAVIS_OS_NAME" = "osx" ]; then export PATH=/usr/bin:${PATH}; fi
+    mkdir -p build
+    cd build
+    cmake ..
+    make -j 4
+    sudo make install
+    fmm
+    stmatch
+    ubodt_gen
+    cd ../example/python
+    which python
+    python fmm_test.py
+  fi
 branches:
   only:
   - master

appveyor.yml

@@ -19,6 +19,7 @@ init:
     - '%CYG_SH% "install apt-cyg /bin"'
     - '%CYG_SH% "apt-cyg install make gcc-g++ cmake gdal libboost-devel libgdal-devel"'
     - '%CYG_SH% "apt-cyg install swig python-devel"'
+    - '%CYG_SH% "python -m pip install setuptools wheel"'
 
 # clone directory
 clone_folder: C:/cygwin64/home/appveyor/fmm

build_manylinux_wheel.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Note: this script should run in manylinux2010 docker container. Please
+# use docker/Dockerfile.manylinux2010 to build the image.
+
+set -e
+
+script_file=$(readlink -f $0)
+script_dir=$(dirname $script_file)
+cd "$script_dir"
+
+version=$1
+if [ -n "$version" ]; then
+    echo "Rewriting python/_version.py"
+    sed -i s"/^__version__\s*=.*/__version__ = '$version'/" python/_version.py
+fi
+
+echo "Building FMM package for cp38-cp38"
+export PATH=/opt/python/cp38-cp38/bin:$PATH
+mkdir -p build && cd build
+cmake .. -DPYTHON_INCLUDE_DIR=$(python -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") -DPYTHON_LIBRARY=$(python -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))")
+make -j2
+make install
+auditwheel repair python/dist/*.whl
+
+echo "Building FMM package for cp37-cp37m"
+export PATH=/opt/python/cp37-cp37m/bin:$PATH
+rm -rf python
+cmake .. -DPYTHON_INCLUDE_DIR=$(python -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") -DPYTHON_LIBRARY=$(python -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))")
+make
+auditwheel repair python/dist/*.whl
+
+echo "Building FMM package for cp36-cp36m"
+export PATH=/opt/python/cp36-cp36m/bin:$PATH
+rm -rf python
+cmake .. -DPYTHON_INCLUDE_DIR=$(python -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") -DPYTHON_LIBRARY=$(python -c "import distutils.sysconfig as sysconfig; print(sysconfig.get_config_var('LIBDIR'))")
+make
+auditwheel repair python/dist/*.whl

docker/Dockerfile.manylinux2010

@@ -0,0 +1,42 @@
+FROM quay.io/pypa/manylinux2010_x86_64:latest
+
+RUN yum install -y wget curl-devel zlib-devel libzstd-devel libwebp-devel hdf5-devel expat-devel
+
+# Install sqlite3
+RUN mkdir -p /build/sqlite3 && \
+    cd /build/sqlite3 && \
+    wget https://www.sqlite.org/2021/sqlite-autoconf-3360000.tar.gz && \
+    tar xf sqlite-autoconf-3360000.tar.gz && \
+    cd sqlite-autoconf-3360000 && \
+    CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA=1" ./configure --prefix=/usr/local && \
+    make && make install && ldconfig
+
+# Install boost
+RUN mkdir -p /build/boost && \
+    cd /build/boost && \
+    wget https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \
+    tar xf boost_1_71_0.tar.bz2 && \
+    cd boost_1_71_0 && \
+    ./bootstrap.sh --prefix=/usr/local && \
+    ./b2 install --prefix=/usr/local --with=all || true && \
+    ldconfig  # some modules may fail to install, but it does not matter
+
+# Install proj6
+RUN mkdir -p /build/proj6 && \
+    cd /build/proj6 && \
+    wget https://download.osgeo.org/proj/proj-6.1.1.tar.gz && \
+    tar xf proj-6.1.1.tar.gz && \
+    cd proj-6.1.1 && \
+    ./configure --prefix=/usr/local && \
+    make -j4 && make install && ldconfig
+
+# Install gdal
+RUN mkdir -p /build/gdal && \
+    cd /build/gdal && \
+    wget https://download.osgeo.org/gdal/2.4.4/gdal-2.4.4.tar.gz && \
+    tar xf gdal-2.4.4.tar.gz && \
+    cd gdal-2.4.4 && \
+    ./configure --prefix=/usr/local && \
+    make -j4 && make install && \
+    strip /usr/local/lib/libgdal.so.20 && ldconfig && \
+    cd / && rm -rf /build

python/CMakeLists.txt

@@ -18,9 +18,15 @@ else()
 endif()
 
 execute_process(
-  COMMAND python -c "from distutils.sysconfig import get_python_lib;print(get_python_lib())"
-  OUTPUT_VARIABLE PYTHON_SITE_PACKAGES OUTPUT_STRIP_TRAILING_WHITESPACE)
-message(STATUS "Python packages ${PYTHON_SITE_PACKAGES}")
+  COMMAND python -c "from setup import get_wheel_name;print(get_wheel_name())"
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  OUTPUT_VARIABLE WHEEL_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+set(PYTHON_SOURCES
+  ${CMAKE_CURRENT_SOURCE_DIR}/setup.py
+  ${CMAKE_CURRENT_SOURCE_DIR}/_version.py
+  ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py)
+file(COPY ${PYTHON_SOURCES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
 
 set_source_files_properties(
 ${PROJECT_SOURCE_DIR}/python/fmm.i PROPERTIES CPLUSPLUS ON)
@@ -29,15 +35,29 @@ if (${CMAKE_VERSION} VERSION_LESS 3.13.0)
   message(STATUS "Using swig add module")
   SWIG_ADD_MODULE(fmm python ${PROJECT_SOURCE_DIR}/python/fmm.i)
   swig_link_libraries(fmm FMMLIB ${PYTHON_LIBRARIES})
-  install(TARGETS ${SWIG_MODULE_fmm_REAL_NAME} DESTINATION ${PYTHON_SITE_PACKAGES})
+  add_custom_target(${WHEEL_NAME} ALL
+    DEPENDS ${SWIG_MODULE_fmm_REAL_NAME}
+    ${CMAKE_CURRENT_BINARY_DIR}/setup.py
+    ${CMAKE_CURRENT_BINARY_DIR}/_version.py
+    ${CMAKE_CURRENT_BINARY_DIR}/__init__.py
+    COMMAND python setup.py bdist_wheel
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Building wheel")
 else()
   message(STATUS "Using swig add library")
   SWIG_ADD_LIBRARY(pyfmm
     LANGUAGE python
     SOURCES ${PROJECT_SOURCE_DIR}/python/fmm.i)
   set_property(TARGET pyfmm PROPERTY OUTPUT_NAME fmm)
   swig_link_libraries(pyfmm FMMLIB ${PYTHON_LIBRARIES})
-  install(TARGETS pyfmm DESTINATION ${PYTHON_SITE_PACKAGES})
+  add_custom_target(${WHEEL_NAME} ALL
+    DEPENDS pyfmm
+    ${CMAKE_CURRENT_BINARY_DIR}/setup.py
+    ${CMAKE_CURRENT_BINARY_DIR}/_version.py
+    ${CMAKE_CURRENT_BINARY_DIR}/__init__.py
+    COMMAND python setup.py bdist_wheel
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Building wheel")
 endif()
 
-install(FILES ${CMAKE_BINARY_DIR}/python/fmm.py DESTINATION ${PYTHON_SITE_PACKAGES})
+install(CODE "execute_process(COMMAND python -m pip install --force-reinstall ${CMAKE_CURRENT_BINARY_DIR}/dist/${WHEEL_NAME})")

python/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+
+from .fmm import *
+from ._version import __version__

python/_version.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+__version__ = '0.1.0.dev0'

python/setup.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+from setuptools.dist import Distribution
+from setuptools.command.install import install
+from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
+from _version import __version__
+
+
+# Force building a non-pure (py3-none-any) wheel when extensions were not
+# specified in setup configs. What we want is a wheel with platform-specific
+# tag such as cp38-cp38-linux_x86_64.
+class bdist_wheel(_bdist_wheel):
+    def finalize_options(self):
+        _bdist_wheel.finalize_options(self)
+        self.root_is_pure = False
+
+
+# Workaround a distutils bug, please refer to https://github.com/google/or-tools/issues/616
+class InstallPlatlib(install):
+    def finalize_options(self):
+        install.finalize_options(self)
+        self.install_lib = self.install_platlib
+
+
+def wheel_name(**kwargs):
+    dist = Distribution(attrs=kwargs)
+    bdist_wheel_cmd = dist.get_command_obj('bdist_wheel')
+    bdist_wheel_cmd.ensure_finalized()
+    distname = bdist_wheel_cmd.wheel_dist_name
+    tag = '-'.join(bdist_wheel_cmd.get_tag())
+    return '%s-%s.whl' % (distname, tag)
+
+
+def get_wheel_name():
+    return wheel_name(**setup_args)
+
+
+setup_args = dict(
+    name='FastMM',
+    version=__version__,
+    description='Fast map matching, an open source framework in C++',
+    url='https://github.com/cyang-kth/fmm',
+    packages=['fmm'],
+    package_dir={'fmm': '.'},
+    package_data={'fmm': ['*.so', '*.dll', '*.dylib']},
+    cmdclass={'bdist_wheel': bdist_wheel, 'install': InstallPlatlib})
+
+
+if __name__ == '__main__':
+    setup(**setup_args)