Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding setup.py for conventional packaging/installation #56

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[submodule "ThirdParty/pybind11"]
path = ThirdParty/pybind11
url = https://github.com/pybind/pybind11.git
[submodule "ThirdParty/tbb"]
path = ThirdParty/tbb
url = https://github.com/wjakob/tbb.git
[submodule "ThirdParty/pagmo2"]
path = ThirdParty/pagmo2
url = https://github.com/apc-llc/pagmo2.git
27 changes: 21 additions & 6 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -150,14 +150,29 @@ endif()

# Find the dependencies.

# TBB.
# TODO This should actually be a submodule of pagmo.
# The only reason to have it here is to avoid forking pagmo as well.
set(TBB_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(TBB_BUILD_SHARED ON CACHE BOOL "" FORCE)
set(TBB_BUILD_STATIC OFF CACHE BOOL "" FORCE)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/tbb EXCLUDE_FROM_ALL)
set(TBB_FOUND TRUE)
set(TBB_LIBRARIES tbb tbb_interface)
set(TBB_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/tbb/include)
add_library(TBB::tbb ALIAS tbb)
add_library(TBB::tbb_interface ALIAS tbb_interface)
install(TARGETS ${TBB_LIBRARIES} EXPORT pagmo_export
LIBRARY DESTINATION ${CMAKE_CURRENT_BINARY_DIR}
ARCHIVE DESTINATION ${CMAKE_CURRENT_BINARY_DIR}
RUNTIME DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

# pagmo.
# NOTE: put the minimum version in a variable
# so that we can re-use it below.
set (_PYGMO_MIN_PAGMO_VERSION 2.13.0)
find_package(pagmo REQUIRED)
if(${pagmo_VERSION} VERSION_LESS ${_PYGMO_MIN_PAGMO_VERSION})
message(FATAL_ERROR "The minimum pagmo version required by pygmo is ${_PYGMO_MIN_PAGMO_VERSION}, but version ${pagmo_VERSION} was found instead.")
endif()
set(_PYGMO_MIN_PAGMO_VERSION 2.13.0)
option(PAGMO_BUILD_STATIC_LIBRARY "" TRUE)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/pagmo2 EXCLUDE_FROM_ALL)

# python.
include(YACMAPythonSetup)
Expand All @@ -171,7 +186,7 @@ if(${PYTHON_VERSION_MAJOR} LESS 3 OR (${PYTHON_VERSION_MAJOR} EQUAL 3 AND ${PYTH
endif()

# pybind11.
find_package(pybind11 REQUIRED)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/pybind11)

# Configure the sphinx config file.
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/doc/conf.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/doc/conf.py" @ONLY)
Expand Down
19 changes: 15 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
pygmo
=====
# pygmo

[![Build Status](https://img.shields.io/circleci/project/github/esa/pygmo2/master.svg?style=for-the-badge)](https://circleci.com/gh/esa/pygmo2)
[![Build Status](https://img.shields.io/travis/esa/pygmo2/master.svg?logo=travis&style=for-the-badge)](https://travis-ci.org/esa/pygmo2)
Expand All @@ -22,11 +21,23 @@ at [this link](https://doi.org/10.5281/zenodo.1045336).

The full documentation can be found [here](https://esa.github.io/pygmo2/).

Upgrading from pygmo 1.x.x
==========================
## Building and installing from source

```
pip3 install git+https://github.com/esa/pygmo2.git
```

## Quick testing

```
python3 -c "import pygmo; pygmo.test.run_test_suite(1); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
```

## Upgrading from pygmo 1.x.x

If you were using the old pygmo, have a look here on some technical data on what and why a completely new API
and code was developed: https://github.com/esa/pagmo2/wiki/From-1.x-to-2.x

You will find many tutorials in the documentation, we suggest to skim through them to realize the differences.
The new pygmo (version 2) should be considered (and is) as an entirely different code.

1 change: 1 addition & 0 deletions ThirdParty/pagmo2
Submodule pagmo2 added at 7c502a
1 change: 1 addition & 0 deletions ThirdParty/pybind11
Submodule pybind11 added at 227170
1 change: 1 addition & 0 deletions ThirdParty/tbb
Submodule tbb added at 806df7
13 changes: 8 additions & 5 deletions pygmo/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ foreach(PYGMO_PYTHON_FILE ${PYGMO_PYTHON_FILES})
endforeach()

# Core module.
YACMA_PYTHON_MODULE(core
# Note that LTO causes link time errors with GCC.
# To avoid this, we disable LTO for pybind using NO_EXTRAS.
# For more details, see e.g. https://github.com/BlueBrain/nmodl/issues/266
PYBIND11_ADD_MODULE(core MODULE NO_EXTRAS
core.cpp
common_utils.cpp
common_base.cpp
Expand All @@ -77,9 +80,9 @@ YACMA_PYTHON_MODULE(core
handle_thread_py_exception.cpp
)

target_link_libraries(core PRIVATE Pagmo::pagmo Boost::boost Boost::serialization)
target_include_directories(core SYSTEM PRIVATE "${pybind11_INCLUDE_DIR}")
target_compile_definitions(core PRIVATE "${pybind11_DEFINITIONS}")
target_include_directories(core SYSTEM PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../ThirdParty/pagmo2/include")
target_include_directories(core SYSTEM PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/../ThirdParty/pagmo2/include")
target_link_libraries(core PRIVATE pagmo Boost::boost Boost::serialization pybind11::pybind11)
target_compile_options(core PRIVATE
"$<$<CONFIG:Debug>:${PYGMO_CXX_FLAGS_DEBUG}>"
"$<$<CONFIG:Release>:${PYGMO_CXX_FLAGS_RELEASE}>"
Expand Down Expand Up @@ -115,7 +118,7 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.9.0")
endif()

# Setup the installation path.
set(PYGMO_INSTALL_PATH "${YACMA_PYTHON_MODULES_INSTALL_PATH}/pygmo")
set(PYGMO_INSTALL_PATH .)

# Add submodules directories
add_subdirectory(plotting)
Expand Down
270 changes: 270 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
"""Setup for the ESA software components
Copyright (C) 2015 Ehsan Azar ([email protected])
Copyright (C) 2019, 2020 Dmitry Mikushin ([email protected])

This file basically just uses CMake to compile the given project.

To build the package:
python3 setup.py build
To build and install:
python3 setup.py install
To package the wheel (after pip installing twine and wheel):
python3 setup.py bdist_wheel
To upload the binary wheel to PyPi
twine upload dist/*.whl
To upload the source distribution to PyPi
python3 setup.py sdist
twine upload <project-name>/<project-name>-*.tar.gz
To exclude certain options in the cmake config use --no:
for example:
--no USE_AVX_INSTRUCTIONS: will set -DUSE_AVX_INSTRUCTIONS=no
Additional options:
--compiler-flags: pass flags onto the compiler, e.g. --compiler-flags "-Os -Wall" passes -Os -Wall onto GCC.
-G: Set the CMake generator. E.g. -G "Visual Studio 14 2015"
--clean: delete any previous build folders and rebuild. You should do this if you change any build options
by setting --compiler-flags or --no since the last time you ran a build. This will
ensure the changes take effect.
--set: set arbitrary cmake options e.g. --set CUDA_HOST_COMPILER=/usr/bin/gcc-6.4.0
passes -DCUDA_HOST_COMPILER=/usr/bin/gcc-6.4.0 to CMake.
"""

project_name = "pygmo"

import os
import re
import sys
import shutil
import platform
import subprocess
import multiprocessing
from distutils import log, dir_util
from math import ceil,floor

import setuptools
from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext
from setuptools.command.install_lib import install_lib
from distutils.version import LooseVersion

from setuptools.command.test import test as TestCommand

def get_extra_cmake_options():
"""read --clean, --no, --set, --compiler-flags, and -G options from the command line and add them as cmake switches.
"""
_cmake_extra_options = []
_clean_build_folder = False

opt_key = None

has_generator = False

argv = [arg for arg in sys.argv] # take a copy
# parse command line options and consume those we care about
for arg in argv:
if opt_key == 'compiler-flags':
_cmake_extra_options.append('-DCMAKE_CXX_FLAGS={arg}'.format(arg=arg.strip()))
elif opt_key == 'G':
has_generator = True
_cmake_extra_options += ['-G', arg.strip()]
elif opt_key == 'no':
_cmake_extra_options.append('-D{arg}=no'.format(arg=arg.strip()))
elif opt_key == 'set':
_cmake_extra_options.append('-D{arg}'.format(arg=arg.strip()))

if opt_key:
sys.argv.remove(arg)
opt_key = None
continue

if arg == '--clean':
_clean_build_folder = True
sys.argv.remove(arg)
continue

if arg == '--yes':
print("The --yes options to setup.py don't do anything since all these options ")
print("are on by default. So --yes has been removed. Do not give it to setup.py.")
sys.exit(1)
if arg in ['--no', '--set', '--compiler-flags']:
opt_key = arg[2:].lower()
sys.argv.remove(arg)
continue
if arg in ['-G']:
opt_key = arg[1:]
sys.argv.remove(arg)
continue

# If no explicit CMake Generator specification,
# prefer Ninja on Windows
if (not has_generator) and (platform.system() == "Windows") and shutil.which("ninja"):
_cmake_extra_options += ['-G', "Ninja"]

return _cmake_extra_options, _clean_build_folder

cmake_extra_options,clean_build_folder = get_extra_cmake_options()


class CMakeExtension(Extension):
def __init__(self, name, sourcedir=''):
Extension.__init__(self, name, sources=[])
self.sourcedir = os.path.abspath(sourcedir)

def rmtree(name):
"""remove a directory and its subdirectories.
"""
def remove_read_only(func, path, exc):
excvalue = exc[1]
if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
func(path)
else:
raise

if os.path.exists(name):
log.info('Removing old directory {}'.format(name))
shutil.rmtree(name, ignore_errors=False, onerror=remove_read_only)


class CMakeBuild(build_ext):

def get_cmake_version(self):
try:
out = subprocess.check_output(['cmake', '--version'])
except OSError:
raise RuntimeError("\n*******************************************************************\n" +
" CMake must be installed to build the following extensions: " +
", ".join(e.name for e in self.extensions) +
"\n*******************************************************************\n")
return re.search(r'version\s*([\d.]+)', out.decode()).group(1)

def run(self):
cmake_version = self.get_cmake_version()
if platform.system() == "Windows":
if LooseVersion(cmake_version) < '3.1.0':
raise RuntimeError("CMake >= 3.1.0 is required on Windows")

for ext in self.extensions:
self.build_extension(ext)

def build_extension(self, ext):
cmake_args = []

cmake_args += cmake_extra_options

cfg = 'Debug' if self.debug else 'Release'
build_args = ['--config', cfg]

if platform.system() != "Windows":
cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
# Do a parallel build
build_args += ['--', '-j'+str(num_available_cpu_cores(2))]

build_folder = os.path.abspath(self.build_temp)

if clean_build_folder:
rmtree(build_folder)
if not os.path.exists(build_folder):
os.makedirs(build_folder)

cmake_setup = ['cmake', ext.sourcedir] + cmake_args
cmake_build = ['cmake', '--build', '.'] + build_args

print("Building extension for Python {}".format(sys.version.split('\n',1)[0]))
print("Invoking CMake setup: '{}'".format(' '.join(cmake_setup)))
sys.stdout.flush()
subprocess.check_call(cmake_setup, cwd=build_folder)
print("Invoking CMake build: '{}'".format(' '.join(cmake_build)))
sys.stdout.flush()
subprocess.check_call(cmake_build, cwd=build_folder)

class CMakeInstall(install_lib):

def install(self):

build_cmd = self.get_finalized_command('build_ext')
build_files = build_cmd.get_outputs()
build_temp = getattr(build_cmd, 'build_temp')

install_dir = os.path.join(os.path.abspath(self.install_dir), project_name)

cmake_install_prefix = ['cmake', '-DCMAKE_INSTALL_PREFIX=' + install_dir, '-P', 'cmake_install.cmake' ]

# Adjust install prefix as shown at LLVM and not widely known:
# https://llvm.org/docs/CMake.html#id6
print("Adjusting CMake install prefix: '{}'".format(' '.join(cmake_install_prefix)))
sys.stdout.flush()
subprocess.check_call(cmake_install_prefix, cwd=build_temp)

def num_available_cpu_cores(ram_per_build_process_in_gb):
if 'TRAVIS' in os.environ and os.environ['TRAVIS']=='true':
# When building on travis-ci, just use 2 cores since travis-ci limits
# you to that regardless of what the hardware might suggest.
return 2
try:
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
mem_gib = mem_bytes/(1024.**3)
num_cores = multiprocessing.cpu_count()
# make sure we have enough ram for each build process.
mem_cores = int(floor(mem_gib/float(ram_per_build_process_in_gb)+0.5));
# We are limited either by RAM or CPU cores. So pick the limiting amount
# and return that.
return max(min(num_cores, mem_cores), 1)
except ValueError:
return 2 # just assume 2 if we can't get the os to tell us the right answer.

class PyTest(TestCommand):
user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")]

def initialize_options(self):
TestCommand.initialize_options(self)
self.pytest_args = '--ignore docs --ignore ' + os.path.join('ThirdParty', project_name)

def run_tests(self):
import shlex
#import here, cause outside the eggs aren't loaded
import pytest
errno = pytest.main(shlex.split(self.pytest_args))
sys.exit(errno)

def read_version_from_cmakelists(cmake_file):
"""Read version information
"""
major = re.findall("set\(CPACK_PACKAGE_VERSION_MAJOR.*\"(.*)\"", open(cmake_file).read())[0]
minor = re.findall("set\(CPACK_PACKAGE_VERSION_MINOR.*\"(.*)\"", open(cmake_file).read())[0]
patch = re.findall("set\(CPACK_PACKAGE_VERSION_PATCH.*\"(.*)\"", open(cmake_file).read())[0]
return major + '.' + minor + '.' + patch

def read_entire_file(fname):
"""Read text out of a file relative to setup.py.
"""
return open(os.path.join(fname)).read()

with open("README.md", "r") as fh:
long_description = fh.read()

setup(
name=project_name,
#version=read_version_from_cmakelists('CMakeLists.txt'),
version="2.15.0",
author="Dario Izzo",
author_email="[email protected]",
description="A platform to perform parallel computations of optimisation tasks (global and local) via the asynchronous generalized island model.",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/esa/pygmo2",
packages=setuptools.find_packages(),
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
],
python_requires='>=3.6',
license='MPL 2.0',
ext_modules=[CMakeExtension(os.path.join('ThirdParty', project_name))], #'tools/python')],
cmdclass=dict(build_ext=CMakeBuild, install_lib=CMakeInstall), #, test=PyTest),
zip_safe=False,
tests_require=[],
# removed 'cmake' because the pip cmake package is busted, maybe someday it will be usable.
install_requires=['numpy', 'cloudpickle', 'networkx', 'dill', 'numba', 'numba', 'ipyparallel'],
keywords=['optimization', 'parallel computations', 'island model'],
)