Homebrew building and updated ciwheelbuild

.github/workflows/python-publish.yml

@@ -9,9 +9,10 @@ jobs:
     name: Build on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.7]
+        python-version: [3.8]
 
     steps:
       - uses: actions/checkout@v2
@@ -24,26 +25,14 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: macOS - Prepare MacPorts - Prefer Xcode 11.5
-        if: ${{ runner.os == 'macOS' }}
-        run: |
-          xcode-select -p
-          xcodebuild -version
-          sudo xcode-select -s /Applications/Xcode_11.5.app/Contents/Developer
-          xcodebuild -version
-
-      - name: macOS - Install MacPorts
-        if: ${{ runner.os == 'macOS' }}
-        uses: WildbookOrg/wbia-pypkg-build/actions/[email protected]
-
       - name: Build wheel
         env:
-          CIBW_SKIP: cp27-* pp27-* pp36-* *-win32 *-win_amd64 *-manylinux_i686 *-manylinux_aarch64 *-manylinux_ppc64le *-manylinux_s390x
-          CIBW_BEFORE_BUILD_LINUX: bash scripts/ci_prepare_centos_for_build.sh
+          CIBW_SKIP: pp* cp36-* *_i686 *_ppc64le *_s390x cp36-musllinux_* cp37-musllinux_* cp38-musllinux_* cp39-musllinux_* cp310-musllinux_*
+          CIBW_BEFORE_BUILD_LINUX: bash scripts/ci_prepare_linux_for_build.sh
           CIBW_BEFORE_BUILD_MACOS: bash scripts/ci_prepare_macos_for_build.sh
           CIBW_TEST_COMMAND: python -c "import pyflann; from pyflann.__main__ import main; main()"
         run: |
-          python -m pip install cibuildwheel==1.4.2
+          python -m pip install cibuildwheel
           python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v2

.github/workflows/testing.yml

@@ -0,0 +1,72 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Testing
+
+on: push
+
+jobs:
+  lint:
+    name: Lint code
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: '3.8'
+
+      # Lint things before going any further
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --show-source --statistics
+          # exit-zero treats all errors as warnings.
+          flake8 . --count --exit-zero --max-complexity=10 --statistics
+
+  test:
+    name: Test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      max-parallel: 4
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, ubuntu-18.04, macos-latest, macos-11]
+        python-version: [3.7, 3.8, 3.9, '3.10']
+
+    steps:
+      # Checkout and env setup
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies (Linux)
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          sudo bash scripts/ci_prepare_linux_for_build.sh
+
+      - name: Install dependencies (macOS)
+        if: ${{ runner.os == 'macOS' }}
+        run: |
+          bash scripts/ci_prepare_macos_for_build.sh
+
+      - name: Build project
+        run: |
+          bash run_developer_setup.sh
+
+      # Install and test
+      - name: Install project
+        run: |
+          pip install -e .[tests]
+
+      - name: Test project
+        run: |
+          pytest

CMake/FindLZ4.cmake

@@ -0,0 +1,39 @@
+###############################################################################
+# Find LZ4
+#
+# This sets the following variables:
+# LZ4_FOUND        - True if LZ4 was found.
+# LZ4_INCLUDE_DIRS - Directories containing the LZ4 include files.
+# LZ4_LIBRARIES    - Libraries needed to use LZ4.
+# LZ4_LIBRARY      - Library needed to use LZ4.
+# LZ4_LIBRARY_DIRS - Library needed to use LZ4.
+
+find_package(PkgConfig REQUIRED)
+
+# If found, LZ$_* variables will be defined
+pkg_check_modules(LZ4 REQUIRED liblz4)
+
+if(NOT LZ4_FOUND)
+    find_path(LZ4_INCLUDE_DIR lz4.h
+      HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+      PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+      PATH_SUFFIXES include)
+
+    find_library(LZ4_LIBRARY
+      NAMES lz4 lz4_static
+      HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
+      PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
+      PATH_SUFFIXES lib)
+
+    if(LZ4_LIBRARY)
+        set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+        get_filename_component(LZ4_LIBRARY_DIRS ${LZ4_LIBRARY} DIRECTORY)
+    endif()
+else()
+    find_library(LZ4_LIBRARY
+        NAMES lz4 lz4_static
+        PATHS ${LZ4_LIBRARY_DIRS}
+        NO_DEFAULT_PATH)
+endif()
+
+mark_as_advanced(LZ4_LIBRARY LZ4_INCLUDE_DIRS LZ4_LIBRARY_DIRS LZ4_LIBRARIES)

CMake/pyflann_utils.cmake

@@ -23,6 +23,7 @@ macro(DISSECT_VERSION)
 endmacro(DISSECT_VERSION)
 
 # workaround a FindHDF5 bug
+set(HDF5_PREFER_PARALLEL TRUE)
 macro(find_hdf5)
   find_package(HDF5)
 

CMakeLists.txt

@@ -37,12 +37,15 @@ dissect_version()
 get_os_info()
 
 if(OS_IS_MACOS)
-  message(STATUS "INCLUDING MACPORTS")
+  message(STATUS "INCLUDING HOMEBREW")
 
-  include_directories(/opt/local/include)
-  link_directories(/opt/local/lib)
+  execute_process(COMMAND brew --prefix OUTPUT_VARIABLE BREW_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE)
+  message(STATUS "BREW_PREFIX = ${BREW_PREFIX}")
 
-  set(CMAKE_INSTALL_RPATH "/opt/local/lib")
+  include_directories("${BREW_PREFIX}/include")
+  link_directories("${BREW_PREFIX}/lib")
+
+  set(CMAKE_INSTALL_RPATH "${BREW_PREFIX}/lib")
 
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -stdlib=libc++")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
@@ -94,11 +97,11 @@ set(TEST_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/tests)
 
 option(BUILD_C_BINDINGS "Build C bindings" ON)
 option(BUILD_CUDA_LIB "Build CUDA library" OFF)
-option(BUILD_EXAMPLES "Build examples" ON)
-option(BUILD_TESTS "Build tests" ON)
-option(BUILD_DOC "Build documentation" ON)
+option(BUILD_EXAMPLES "Build examples" OFF)
+option(BUILD_TESTS "Build tests" OFF)
+option(BUILD_DOC "Build documentation" OFF)
 option(USE_OPENMP "Use OpenMP multi-threading" ON)
-option(USE_MPI "Use MPI" OFF)
+option(USE_MPI "Use MPI" ON)
 
 set(NVCC_COMPILER_BINDIR
     ""
@@ -184,7 +187,7 @@ endif(BUILD_CUDA_LIB)
 
 find_package(PkgConfig REQUIRED)
 
-pkg_check_modules(LZ4 REQUIRED liblz4)
+find_package(LZ4)
 
 # set the C/C++ include path to the "include" directory
 include_directories(BEFORE ${PROJECT_SOURCE_DIR}/src/cpp)
@@ -297,4 +300,5 @@ message(STATUS "Building documentation: ${BUILD_DOC}")
 message(STATUS "Building matlab bindings: ${BUILD_MATLAB_BINDINGS}")
 message(STATUS "Building CUDA library: ${BUILD_CUDA_LIB}")
 message(STATUS "Using OpenMP support: ${USE_OPENMP}")
+message(STATUS "HDF5 MPI: ${HDF5_IS_PARALLEL}")
 message(STATUS "Using MPI support: ${USE_MPI}")

pyflann/flann_ctypes.py

@@ -50,9 +50,9 @@
 
 class CustomStructure(Structure):
     """
-        This class extends the functionality of the ctype's structure
-        class by adding custom default values to the fields and a way of translating
-        field types.
+    This class extends the functionality of the ctype's structure
+    class by adding custom default values to the fields and a way of translating
+    field types.
     """
 
     _defaults_ = {}

pyflann/index.py

@@ -160,7 +160,7 @@ def __len__(self):
         return self.shape[0]
 
     def get_indexed_shape(self):
-        """ returns the shape of the data being indexed """
+        """returns the shape of the data being indexed"""
         npts, dim = self.__curindex_data.shape
         for _extra in self.__added_data:
             npts += _extra.shape[0]
@@ -584,7 +584,12 @@ def hierarchical_kmeans(
         self.__flann_parameters.update(params)
 
         numclusters = flann.compute_cluster_centers[pts.dtype.type](
-            pts, npts, dim, num_clusters, result, pointer(self.__flann_parameters),
+            pts,
+            npts,
+            dim,
+            num_clusters,
+            result,
+            pointer(self.__flann_parameters),
         )
         if numclusters <= 0:
             raise FLANNException('Error occured during clustering procedure.')

pyproject.toml

@@ -1,2 +1,2 @@
 [build-system]
-requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4", "scikit-build", "cmake", "ninja"]
+requires = ["setuptools", "wheel", "setuptools_scm[toml]", "scikit-build", "cmake", "ninja"]

requirements/build.txt

@@ -1,6 +1,6 @@
 cmake
 ninja
 scikit-build
-setuptools>=42
-setuptools_scm[toml]>=3.4
+setuptools
+setuptools_scm[toml]
 wheel

requirements/tests.txt

@@ -1,6 +1,6 @@
 brunette
-codecov >= 2.0.15
-coverage >= 4.3.4
-pytest >= 5.4.3
+codecov
+coverage
+pytest
 pytest-cov
-xdoctest >= 0.3.0
+xdoctest

run_developer_setup.sh

@@ -9,7 +9,3 @@ python setup.py build_ext --inplace
 python setup.py develop
 
 pip install -e .
-
-# Point to custom opencv
-# python setup.py build_ext --inplace -- -DOpenCV_DIR=$HOME/code/opencv/build
-# python setup.py develop -- -DOpenCV_DIR=$HOME/code/opencv/build

scripts/ci_prepare_linux_for_build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -ex
+
+pip install -r requirements/build.txt
+
+if command -v yum &> /dev/null
+then
+    yum install -y \
+        pkgconfig \
+        boost \
+        boost-thread \
+        boost-devel \
+        libgomp \
+        hdf5-openmpi \
+        lz4-devel
+else
+    apt-get install \
+        pkg-config \
+        libboost-all-dev \
+        libopenmpi-dev \
+        libomp5 \
+        libomp-dev \
+        libhdf5-openmpi-dev \
+        liblz4-dev
+fi