feat: Python wheels workflow and build backend

.gitignore

@@ -4,13 +4,17 @@ branches/
 build/
 dist/
 ext/
+wheelhouse/
+_coverage/
+.python-version
 .idea
 .vscode
 .cproject
 .project
 .DS_Store
 *.pyc
-_coverage/
+/*.lock
+gastest.o
 
 # Exclude test files I tend to leave around at the top level. The leading
 # slash ensures that files with these extensions within subdirectories are not
@@ -21,3 +25,5 @@ _coverage/
 /*.jxl
 /*.tx
 /*.log
+
+

CMakeLists.txt

@@ -250,10 +250,16 @@ if (NOT EMBEDPLUGINS AND NOT BUILD_OIIOUTIL_ONLY)
     endforeach ()
 endif ()
 
-if (USE_PYTHON AND Python3_Development_FOUND AND NOT BUILD_OIIOUTIL_ONLY)
+
+if (WIN32)
+    set (_py_dev_found Python3_Development_FOUND)    
+else ()
+    set (_py_dev_found Python3_Development.Module_FOUND)
+endif ()
+if (USE_PYTHON AND ${_py_dev_found} AND NOT BUILD_OIIOUTIL_ONLY)
     add_subdirectory (src/python)
 else ()
-    message (STATUS "Not building Python bindings: USE_PYTHON=${USE_PYTHON}, Python3_Development_FOUND=${Python3_Development_FOUND}")
+    message (STATUS "Not building Python bindings: USE_PYTHON=${USE_PYTHON}, Python3_Development.Module_FOUND=${Python3_Development.Module_FOUND}")
 endif ()
 
 add_subdirectory (src/include)

INSTALL.md

@@ -357,6 +357,28 @@ produce a dynamic-linked version.
 
 3. Execute the PowerShell command from where vcpkg is located in directory. ``vcpkg install openimageio``
 
+
+Python-based Builds and Installs
+--------------------------------
+
+**Installing from prebuilt binary distributions**
+
+If you're only interested in the Python module and the CLI tools, you can install with `pip` or `uv`:
+
+> ```pip install OpenImageIO```
+
+**Building and installing from source**
+
+If you have a C++ compiler installed, you can also use the Python build-backend to compile and install
+from source by navigating to the root of the repository and running: ```pip install .```
+
+This will download and install CMake and Ninja if necessary, and invoke the CMake build system; which,
+in turn, will build missing dependencies, compile OIIO, and install the Python module, the libraries, 
+the headers, and the CLI tools to a platform-specific, Python-specific location. 
+
+See the [scikit-build-core docs](https://scikit-build-core.readthedocs.io/en/latest/configuration.html#configuring-cmake-arguments-and-defines)
+for more information on customizing and overriding build-tool options and CMake arguments.
+
 Test Images
 -----------
 

pyproject.toml

@@ -0,0 +1,125 @@
+[project]
+name = "OpenImageIO"
+# The build backend ascertains the version from the CMakeLists.txt file.
+dynamic = ["version"]
+description = "Reading, writing, and processing images in a wide variety of file formats, using a format-agnostic API, aimed at VFX applications."
+authors = [
+    {name = "Larry Gritz", email = "[email protected]"},
+    {name = "OpenImageIO Contributors", email = "[email protected]"}
+]
+maintainers = [
+    {name = "OpenImageIO Contributors", email="[email protected]"},
+]
+readme = "README.md"
+license = {text = "Apache-2.0"}
+classifiers = [
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: Apache Software License",
+    "Topic :: Multimedia :: Graphics",
+    "Topic :: Multimedia :: Video",
+    "Topic :: Multimedia :: Video :: Display",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">= 3.8"
+
+[project.urls]
+Homepage = "https://openimageio.org/"
+Source = "https://github.com/AcademySoftwareFoundation/OpenImageIO"
+Documentation = "https://openimageio.readthedocs.io"
+Issues = "https://github.com/AcademySoftwareFoundation/OpenImageIO/issues"
+
+
+[project.scripts]
+# Use the convention below to expose CLI tools as Python scripts.
+iconvert = "OpenImageIO:_command_line"
+idiff = "OpenImageIO:_command_line"
+igrep = "OpenImageIO:_command_line"
+iinfo = "OpenImageIO:_command_line"
+maketx = "OpenImageIO:_command_line"
+oiiotool = "OpenImageIO:_command_line"
+testtex = "OpenImageIO:_command_line"
+
+[build-system]
+build-backend = "scikit_build_core.build"
+requires = [
+    "scikit-build-core>=0.10.6,<1",
+    "pybind11",
+]
+
+[tool.scikit-build]
+build.verbose = true
+# Exclude unnecessary directories from the source distribution.
+sdist.exclude = [".github", "testsuite", "ASWF", "docs"]
+# Pin minimum scikit-build-core to that specified in build-system.requires.
+minimum-version = "build-system.requires"
+# Pin minimum CMake version to that specified in CMakeLists.txt.
+cmake.version = "CMakeLists.txt"
+wheel.license-files = ["LICENSE.md", "THIRD-PARTY.md"]
+# Make sure the package is structured as expected.
+wheel.install-dir = "OpenImageIO"
+
+[tool.scikit-build.cmake.define]
+# Build missing dependencies. See src/cmake for details.
+OpenImageIO_BUILD_MISSING_DEPS = "all"
+# Don't build tests. Dramatically improves build time.
+OIIO_BUILD_TESTS = "0"
+# Prefer linking static dependencies when possible.
+LINKSTATIC = "1"
+
+# Dynamically set the package version metadata by pasrsing CMakeLists.txt.
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "CMakeLists.txt" 
+regex = 'set \(OpenImageIO_VERSION "(?P<value>[0-9a-z.]+)"\)'
+
+
+[tool.cibuildwheel]
+build-verbosity = 1
+skip = [
+    # Skip 32-bit builds
+    "*-win32",
+    "*-manylinux_i686",
+    # Building with musl seems to work, but the repair-wheel step seems to fail...
+    # This may be a bug in repairwheel (or auditwheel)?
+    "*musllinux*",  
+]
+test-command = "oiiotool --buildinfo"
+
+[tool.cibuildwheel.macos.environment]
+# Specify a persistent build directory so the repairwheel command can (re)find
+# dependency libraries built by OpenImageIO.
+SKBUILD_BUILD_DIR = "/tmp/build_oiio_wheels"
+# Always (re)build the TIFF library; see #4439 for more details.
+SKBUILD_CMAKE_ARGS = "-DOpenImageIO_BUILD_LOCAL_DEPS=TIFF; -DOpenImageIO_BUILD_MISSING_DEPS=all"
+# C++17 - std::filesystem is only available in macOS 10.15 and later.
+MACOSX_DEPLOYMENT_TARGET = "10.15"
+# Optimize for size (not speed).
+SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
+
+[tool.cibuildwheel.linux.environment]
+SKBUILD_BUILD_DIR = "/tmp/build_oiio_wheels"
+SKBUILD_CMAKE_ARGS = "-DOpenImageIO_BUILD_LOCAL_DEPS=all; -DOpenImageIO_BUILD_MISSING_DEPS=all"
+SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
+# Suppress warnings that cause linux cibuildwheel build to fail
+CXXFLAGS = "-Wno-error=stringop-overflow -Wno-pragmas"
+
+[tool.cibuildwheel.windows.environment]
+SKBUILD_CMAKE_BUILD_TYPE = "MinSizeRel"
+
+# On macOS and Linux, run a custom repair-wheel-command after the build.
+# See tasks.py for more details.
+[tool.cibuildwheel.macos]
+before-build = "pip install repairwheel invoke"
+repair-wheel-command = "invoke wheel-repair --build-dir ${SKBUILD_BUILD_DIR} --wheel-path {wheel} --output-dir {dest_dir}"
+
+[tool.cibuildwheel.linux]
+before-build = "pip install repairwheel invoke"
+repair-wheel-command = "invoke wheel-repair --build-dir ${SKBUILD_BUILD_DIR} --wheel-path {wheel} --output-dir {dest_dir}"

src/build-scripts/build_libtiff.bash

@@ -45,6 +45,7 @@ if [[ -z $DEP_DOWNLOAD_ONLY ]]; then
                -Dtiff-tests=${LIBTIFF_BUILD_TESTS:-OFF} \
                -Dtiff-docs=${LIBTIFF_BUILD_TESTS:-OFF} \
                -Dlibdeflate=ON \
+               -Dwebp=OFF \ 
                ${LIBTIFF_BUILDOPTS}
     time cmake --build build --target install
 fi

src/cmake/build_libdeflate.cmake

@@ -9,7 +9,7 @@
 set_cache (libdeflate_BUILD_VERSION 1.20 "libdeflate version for local builds")
 set (libdeflate_GIT_REPOSITORY "https://github.com/ebiggers/libdeflate")
 set (libdeflate_GIT_TAG "v${libdeflate_BUILD_VERSION}")
-set_cache (libdeflate_BUILD_SHARED_LIBS  ${LOCAL_BUILD_SHARED_LIBS_DEFAULT}
+set_cache (libdeflate_BUILD_SHARED_LIBS OFF # ${LOCAL_BUILD_SHARED_LIBS_DEFAULT}
            DOC "Should a local libdeflate build, if necessary, build shared libraries" ADVANCED)
 
 string (MAKE_C_IDENTIFIER ${libdeflate_BUILD_VERSION} libdeflate_VERSION_IDENT)
@@ -20,6 +20,7 @@ build_dependency_with_cmake(libdeflate
     GIT_TAG         ${libdeflate_GIT_TAG}
     CMAKE_ARGS
         -D BUILD_SHARED_LIBS=${libdeflate_BUILD_SHARED_LIBS}
+        -D LIBDEFLATE_BUILD_SHARED_LIB=${libdeflate_BUILD_SHARED_LIBS}
         -D CMAKE_POSITION_INDEPENDENT_CODE=ON
         -D CMAKE_INSTALL_LIBDIR=lib
         -D LIBDEFLATE_BUILD_GZIP=OFF
@@ -32,6 +33,7 @@ set (libdeflate_ROOT ${libdeflate_LOCAL_INSTALL_DIR})
 # Signal to caller that we need to find again at the installed location
 set (libdeflate_REFIND TRUE)
 set (libdeflate_REFIND_ARGS CONFIG)
+set (libdeflate_REFIND_VERSION ${libdeflate_BUILD_VERSION})
 
 if (libdeflate_BUILD_SHARED_LIBS)
     install_local_dependency_libs (libdeflate libdeflate)

src/cmake/fancy_add_executable.cmake

@@ -53,6 +53,17 @@ macro (fancy_add_executable)
         target_link_libraries (${_target_NAME} PRIVATE ${_target_LINK_LIBRARIES})
         target_link_libraries (${_target_NAME} PRIVATE ${PROFILER_LIBRARIES})
         set_target_properties (${_target_NAME} PROPERTIES FOLDER ${_target_FOLDER})
+        if (SKBUILD) 
+            # When building cli tools with scikit-build, set RPATH
+            # to a relative path to the distribution's lib directory.
+            if (APPLE)
+                set_target_properties (${_target_NAME} PROPERTIES 
+                        INSTALL_RPATH "@loader_path/../${CMAKE_INSTALL_LIBDIR}")
+            elseif (LINUX)
+                set_target_properties (${_target_NAME} PROPERTIES 
+                        INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")
+            endif ()
+        endif ()
         check_is_enabled (INSTALL_${_target_NAME} _target_NAME_INSTALL_enabled)
         if (CMAKE_UNITY_BUILD AND UNITY_BUILD_MODE STREQUAL GROUP)
             set_source_files_properties(${_target_SRC} PROPERTIES

src/cmake/pythonutils.cmake

@@ -32,13 +32,24 @@ macro (find_python)
             list (APPEND _req EXACT)
         endif ()
     endif ()
+
+    # Support building on manylinux docker images, which do not contain
+    # the Development.Embedded component.
+    # https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode
+    if (WIN32)
+        set (_py_components Interpreter Development)
+    else ()
+        set (_py_components Interpreter Development.Module)
+    endif ()
+
     checked_find_package (Python3 ${PYTHON_VERSION}
                           ${_req}
                           VERSION_MIN 3.7
-                          COMPONENTS Interpreter Development
+                          COMPONENTS ${_py_components}
                           PRINT Python3_VERSION Python3_EXECUTABLE
                                 Python3_LIBRARIES
                                 Python3_Development_FOUND
+                                Python3_Development.Module_FOUND
                                 Python3_Interpreter_FOUND )
 
     # The version that was found may not be the default or user
@@ -129,6 +140,19 @@ macro (setup_python_module)
 #                               SUFFIX ".pyd")
 #    endif()
 
+    if (SKBUILD)
+        set (PYTHON_SITE_DIR .)
+        # When building python wheels, set RPATH to a relative path to 
+        # the distribution's lib directory.
+        if (APPLE)
+            set_target_properties (${target_name} PROPERTIES 
+                    INSTALL_RPATH "@loader_path/${CMAKE_INSTALL_LIBDIR}")
+        elseif (LINUX)
+            set_target_properties (${target_name} PROPERTIES
+                    INSTALL_RPATH "$ORIGIN/${CMAKE_INSTALL_LIBDIR}")
+        endif ()
+    endif ()
+
     # In the build area, put it in lib/python so it doesn't clash with the
     # non-python libraries of the same name (which aren't prefixed by "lib"
     # on Windows).

src/python/__init__.py

@@ -2,16 +2,65 @@
 # SPDX-License-Identifier: Apache-2.0
 # https://github.com/AcademySoftwareFoundation/OpenImageIO
 
-import os, sys, platform
+import os
+import sys
+import logging
+import platform
+import subprocess
 
-# This works around the python 3.8 change to stop loading DLLs from PATH on Windows.
-# We reproduce the old behaviour by manually tokenizing PATH, checking that the directories exist and are not ".",
-# then add them to the DLL load path.
-# This behviour can be disabled by setting the environment variable "OIIO_LOAD_DLLS_FROM_PATH" to "0"
-if sys.version_info >= (3, 8) and platform.system() == "Windows" and os.getenv("OIIO_LOAD_DLLS_FROM_PATH", "1") == "1":
-    for path in os.getenv("PATH", "").split(os.pathsep):
-        if os.path.exists(path) and path != ".":
-            os.add_dll_directory(path)
+logger = logging.getLogger(__name__)
 
-from .OpenImageIO import *
+if platform.system() == "Windows":
+    # Python wheel module is dynamically linked to the OIIO DLL present in the bin folder.
+    _here = os.path.abspath(os.path.dirname(__file__))
+    _bin_dir = os.path.join(_here, "bin")
+    if os.path.exists(_bin_dir):
+        if sys.version_info >= (3, 8):
+            os.add_dll_directory(_bin_dir)  
+        else:
+            os.environ['PATH'] = '{0};{1}'.format(_bin_dir, os.getenv('PATH', ''))
+
+    #TODO: Remove. Python 3.8 stopped loading DLLs from PATH on Windows for security reasons.
+    if sys.version_info >= (3, 8):
+        # This works around the python 3.8 change to stop loading DLLs from PATH on Windows.
+        # We reproduce the old behavior by manually tokenizing PATH, checking that the 
+        # directories exist and are not ".", then add them to the DLL load path.
+        # Set the environment variable "OIIO_LOAD_DLLS_FROM_PATH" to "0" to disable this behavior.
+        if os.getenv("OIIO_LOAD_DLLS_FROM_PATH", "1") == "1":
+            logger.warning(
+                "Adding DLL directories to PATH. This behavior will be removed in "
+                "a future version. Set the environment variable OIIO_LOAD_DLLS_FROM_PATH "
+                "to 0 to disable this behavior."
+            )
+            for path in os.getenv("PATH", "").split(os.pathsep):
+                if os.path.exists(path) and path != ".":
+                    os.add_dll_directory(path)
 
+from .OpenImageIO import * # type: ignore # noqa: F401, F403, E402
+
+__version__ = VERSION_STRING # type: ignore # noqa: F405
+
+__status__ = "production"
+
+__author__ = "OpenImageIO Contributors"
+
+__email__ = "[email protected]"
+
+__license__ = "SPDX-License-Identifier: Apache-2.0"
+
+__copyright__ = "Copyright Contributors to the OpenImageIO Project"
+
+__doc__ = """OpenImageIO is a toolset for reading, writing, and manipulating image files of 
+any image file format relevant to VFX / animation via a format-agnostic API 
+with a feature set, scalability, and robustness needed for feature film 
+production.
+"""
+
+
+def _call_program(name, args):
+    bin_dir = os.path.join(os.path.dirname(__file__), 'bin')
+    return subprocess.call([os.path.join(bin_dir, name)] + args)
+
+def _command_line():
+    name = os.path.basename(sys.argv[0])
+    raise SystemExit(_call_program(name, sys.argv[1:]))