diff --git a/.github/workflows/cd-wheel.yml b/.github/workflows/cd-wheel.yml index afe1266..acf28c0 100644 --- a/.github/workflows/cd-wheel.yml +++ b/.github/workflows/cd-wheel.yml @@ -54,7 +54,6 @@ on: # yamllint disable-line rule:truthy - all - Linux - Linux-aarch64 - - Linux-ppc64le - Linux-x86_64 - macOS - macOS-arm64 @@ -76,11 +75,11 @@ jobs: shell: python name: 'setup build matrix' run: | + # setup build matrix # " keys = ("os", "arch", "runner") rows = [ - ("Linux", "aarch64", "ubuntu-22.04"), - ("Linux", "ppc64le", "ubuntu-22.04"), - ("Linux", "x86_64", "ubuntu-22.04"), + ("Linux", "aarch64", "ubuntu-24.04-arm"), + ("Linux", "x86_64", "ubuntu-24.04"), ("macOS", "arm64", "macos-14"), ("macOS", "x86_64", "macos-13"), ] @@ -93,6 +92,7 @@ jobs: import os, json with open(os.getenv("GITHUB_OUTPUT"), "w") as out: print(f"matrix={json.dumps(matrix)}", file=out) + # " build: needs: setup @@ -113,16 +113,21 @@ jobs: if: ${{ runner.os == 'macOS' }} name: setup-macOS run: | + # set macOS deployment target + case $(uname -m) in + arm64) echo MACOSX_DEPLOYMENT_TARGET=11.0 >> $GITHUB_ENV ;; + x86_64) echo MACOSX_DEPLOYMENT_TARGET=10.15 >> $GITHUB_ENV ;; + esac # create gfortran symlink cd $(brew --prefix)/bin gfortran=$(ls gfortran-* | sort | head -n 1) sudo ln -s $gfortran gfortran + # unlink libevent + brew unlink libevent || true # install autotools brew install autoconf brew install automake brew install libtool - # unlink libevent - brew unlink libevent || true # install uv brew install uv @@ -131,20 +136,19 @@ jobs: with: python-version: 3 - - id: setup-qemu - if: ${{ runner.os == 'Linux' }} - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - id: bootstrap run: ./bootstrap.sh - id: source-date-epoch run: | - SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + read -r SOURCE_DATE_EPOCH < source-date-epoch || true + SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-$(git log -1 --pretty=%ct)} echo SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH >> $GITHUB_ENV - echo $(git log -1 --pretty=%ci) [timestamp=$SOURCE_DATE_EPOCH] + test $(uname) = Darwin && (echo ZERO_AR_DATE=1 >> $GITHUB_ENV) + echo [SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH] $( + date -u -d @$SOURCE_DATE_EPOCH 2>/dev/null || + date -u -r $SOURCE_DATE_EPOCH 2>/dev/null ) + working-directory: package/source - id: build uses: pypa/cibuildwheel@v2.22.0 @@ -159,6 +163,8 @@ jobs: CIBW_ARCHS: "${{ matrix.arch }}" CIBW_BEFORE_ALL: >- bash {project}/cibw-build-mpi.sh + CIBW_BEFORE_BUILD: >- + bash {project}/cibw-patch-cmd.sh CIBW_TEST_COMMAND: >- bash {project}/cibw-check-mpi.sh CIBW_ENVIRONMENT_PASS: >- @@ -171,12 +177,8 @@ jobs: SOURCE="$PWD/package/source" WORKDIR="$PWD/package/workdir" DESTDIR="$PWD/package/install" - CIBW_REPAIR_WHEEL_COMMAND_MACOS: > - delocate-wheel - --ignore-missing-dependencies - --exclude libmpi --exclude libpmpi - --require-archs {delocate_archs} - -w {dest_dir} -v {wheel} + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 - id: sha256sum run: | @@ -190,8 +192,24 @@ jobs: - id: upload uses: actions/upload-artifact@v4 with: - name: wheel-${{ inputs.mpiname }}-${{ matrix.os }}-${{ matrix.arch }} + name: "wheel-${{ inputs.mpiname }}-\ + ${{ inputs.version || 'latest' }}-\ + ${{ matrix.os }}-${{ matrix.arch }}" path: wheelhouse/*.whl - id: check - run: ./check-wheel.sh wheelhouse + run: ./wheel-check.sh wheelhouse + + - id: test + run: | + # ./wheel-test.sh + test $(uname) = Linux && runner=( + docker run + -e MPINAME=${{ inputs.mpiname }} + -v $(pwd):/${{ github.workspace }} + -w ${{ github.workspace }} + --platform linux/${{ + matrix.arch == 'aarch64' && 'arm64' || + matrix.arch == 'x86_64' && 'amd64' + }} python:3) + ${runner[@]:-} ./wheel-test.sh wheelhouse diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 9310bdb..a71fdef 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -32,7 +32,6 @@ on: # yamllint disable-line rule:truthy - all - Linux - Linux-aarch64 - - Linux-ppc64le - Linux-x86_64 - macOS - macOS-arm64 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aea16d1..9f68586 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,12 +11,32 @@ on: # yamllint disable-line rule:truthy jobs: - mpich: + mpich-42: uses: ./.github/workflows/cd-wheel.yml with: mpiname: mpich + version: - openmpi: + mpich-41: + uses: ./.github/workflows/cd-wheel.yml + with: + mpiname: mpich + version: 4.1.3 + + mpich-34: + uses: ./.github/workflows/cd-wheel.yml + with: + mpiname: mpich + version: 3.4.3 + + openmpi-50: + uses: ./.github/workflows/cd-wheel.yml + with: + mpiname: openmpi + version: + + openmpi-41: uses: ./.github/workflows/cd-wheel.yml with: mpiname: openmpi + version: 4.1.6 diff --git a/.gitignore b/.gitignore index f5c168b..f654289 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ /venv/ *.egg-info/ *.tar.gz +*.tar.bz2 *~ diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000..53aaf85 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,13 @@ +# https://github.com/adrienverge/yamllint + +extends: default +rules: + document-start: disable + colons: + max-spaces-before: 0 + max-spaces-after: -1 + indentation: + indent-sequences: whatever + line-length: + level: warning + max: 80 diff --git a/LICENSE.md b/LICENSE.md index a6d0ada..1f6eea6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,4 @@ -BSD 2-Clause License - -Copyright (c) 2024, MPI for Python +Copyright (c) 2025, Lisandro Dalcin Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -12,6 +10,10 @@ modification, are permitted provided that the following conditions are met: this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE diff --git a/Makefile b/Makefile index eadfffa..2d73277 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,21 @@ build: ./bootstrap.sh - ./build-wheel.sh dist - ./check-wheel.sh dist + ./wheel-build.sh dist + ./wheel-check.sh dist + ./wheel-test.sh dist lint: codespell *.sh */*.py shellcheck *.sh - ruff check -qn package/*.py + ruff check -qn */*.py yamllint .github/ clean: + $(RM) -r package/METADATA + $(RM) -r package/LICENSE* $(RM) -r package/build - $(RM) -r package/LICENSE - $(RM) -r package/install $(RM) -r package/source $(RM) -r package/workdir + $(RM) -r package/install $(RM) -r package/*.egg-info $(RM) -r .*_cache diff --git a/README.md b/README.md index 8a66e1f..d6f5d73 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This repository builds and publishes [MPICH](https://www.mpich.org) and a variety of - operating systems: *Linux*, *macOS*; -- processor architectures: *AMD64*, *ARM64*, *PPC64*; +- processor architectures: *AMD64*, *ARM64*; - Python implementations: *CPython*, *PyPy*. MPI wheels are uploaded to the [Anaconda.org](https://anaconda.org/mpi4py) diff --git a/bootstrap.sh b/bootstrap.sh index ca2a5f6..a913483 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -8,6 +8,11 @@ case "$mpiname" in esac version=${VERSION:-$version} +ucxversion=1.17.0 +ofiversion=1.22.0 +ucxversion=${UCXVERSION:-$ucxversion} +ofiversion=${OFIVERSION:-$ofiversion} + PROJECT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE=$PROJECT/package SOURCE=$PACKAGE/source @@ -15,40 +20,171 @@ SOURCE=$PACKAGE/source if test "$mpiname" = "mpich"; then urlbase="https://www.mpich.org/static/downloads/$version" tarball="$mpiname-$version.tar.gz" - license=COPYRIGHT fi - if test "$mpiname" = "openmpi"; then urlbase=https://download.open-mpi.org/release/open-mpi/v${version%.*} tarball="$mpiname-$version.tar.gz" - license=LICENSE fi - if test ! -d "$SOURCE"; then if test ! -f "$tarball"; then echo downloading "$urlbase"/"$tarball"... - curl -fsO "$urlbase"/"$tarball" + curl -fsSLO "$urlbase"/"$tarball" else echo reusing "$tarball"... fi - echo extracting "$tarball" to "$SOURCE"... + echo extracting "$tarball"... tar xf "$tarball" mv "$mpiname-$version" "$SOURCE" patch="$PROJECT/patches/$mpiname-$version" if test -f "$patch"; then patch -p1 -i "$patch" -d "$SOURCE" fi - if test "$mpiname-$(uname)" = "openmpi-Darwin"; then - if test -d "$SOURCE"/3rd-party; then - cd "$SOURCE"/3rd-party - tar xf libevent-*.tar.gz && cd libevent-* + if test "$mpiname" = "mpich"; then + if test "${version}" \< "4.2.0"; then + disable_doc='s/^\(install-data-local:\s\+\)\$/\1#\$/' + sed -i.orig "$disable_doc" "$SOURCE"/Makefile.in + fi + fi + if test "$mpiname" = "openmpi"; then + for deptarball in "$SOURCE"/3rd-party/*.tar.*; do + test -f "$deptarball" || continue + echo extracting "$(basename "$deptarball")" + tar xf "$deptarball" -C "$(dirname "$deptarball")" + done + makefiles=( + "$SOURCE"/3rd-party/openpmix/src/util/keyval/Makefile.in + "$SOURCE"/3rd-party/prrte/src/mca/rmaps/rank_file/Makefile.in + "$SOURCE"/3rd-party/prrte/src/util/hostfile/Makefile.in + ) + for makefile in "${makefiles[@]}"; do + test -f "$makefile" || continue + echo "PMIX_CFLAGS_BEFORE_PICKY = @CFLAGS@" >> "$makefile" + echo "PRTE_CFLAGS_BEFORE_PICKY = @CFLAGS@" >> "$makefile" + done + fi + if test "$mpiname" = "openmpi" && test "${version}" \< "5.0.5"; then + if test "$(uname)" = "Darwin" && test -d "$SOURCE"/3rd-party; then + cd "$SOURCE"/3rd-party/libevent-* echo running autogen.sh on "$(basename "$(pwd)")" ./autogen.sh cd "$PROJECT" fi fi + echo writing package metadata ... + echo "Name: $mpiname" > "$PACKAGE/METADATA" + echo "Version: $version" >> "$PACKAGE/METADATA" else echo reusing directory "$SOURCE"... + check() { test "$(awk "/$1/"'{print $2}' "$PACKAGE/METADATA")" = "$2"; } + check Name "$mpiname" || (echo not "$mpiname-$version"!!! && exit 1) + check Version "$version" || (echo not "$mpiname-$version"!!! && exit 1) +fi + +if test "$(uname)" = "Linux"; then + case "$mpiname" in + mpich) MODSOURCE="$SOURCE"/modules ;; + openmpi) MODSOURCE="$SOURCE"/3rd-party ;; + esac + ofigithub="https://github.com/ofiwg/libfabric" + ofiurlbase="$ofigithub/releases/download/v$ofiversion" + ofitarball="libfabric-$ofiversion.tar.bz2" + ofidestdir="$MODSOURCE"/"${ofitarball%%.tar.*}" + if test ! -d "$ofidestdir"; then + if test ! -f "$ofitarball"; then + echo downloading "$ofiurlbase"/"$ofitarball"... + curl -fsSLO "$ofiurlbase"/"$ofitarball" + else + echo reusing "$ofitarball"... + fi + echo extracting "$ofitarball"... + tar xf "$ofitarball" + mkdir -p "$(dirname "$ofidestdir")" + mv "$(basename "$ofidestdir")" "$ofidestdir" + else + echo reusing directory "$ofidestdir"... + fi + ucxgithub="https://github.com/openucx/ucx" + ucxurlbase="$ucxgithub/releases/download/v$ucxversion" + ucxtarball="ucx-$ucxversion.tar.gz" + ucxdestdir="$MODSOURCE"/"${ucxtarball%%.tar.*}" + if test ! -d "$ucxdestdir"; then + if test ! -f "$ucxtarball"; then + echo downloading "$ucxurlbase"/"$ucxtarball"... + curl -fsSLO "$ucxurlbase"/"$ucxtarball" + else + echo reusing "$ucxtarball"... + fi + echo extracting "$ucxtarball"... + tar xf "$ucxtarball" + mkdir -p "$(dirname "$ucxdestdir")" + mv "$(basename "$ucxdestdir")" "$ucxdestdir" + if test "${ucxversion}" \< "1.17.1"; then + cmd='s/\(#include \)/\1\n#include /' + sed -i.orig "$cmd" "$ucxdestdir/src/ucs/time/time.h" + fi + else + echo reusing directory "$ucxdestdir"... + fi +fi + +if test "$mpiname" = "mpich"; then + mpidate=$(sed -nE "s/MPICH_RELEASE_DATE=\"(.*)\"/\1/p" "$SOURCE/configure") +fi +if test "$mpiname" = "openmpi"; then + mpidate=$(sed -nE "s/date=\"(.*)\"/\1/p" "$SOURCE/VERSION") +fi +if test -n "${mpidate+x}"; then + case "$(uname)" in + Linux) + timestamp=$(date -d "$mpidate" "+%s") + ;; + Darwin) + datefmt="%b %d, %Y %T%z" + if test "$mpiname" = "mpich"; then + mpidate=$(awk '{$3=$3",";print $2,$3,$NF}' <<< "$mpidate") + fi + timestamp=$(date -j -f "$datefmt" "$mpidate 12:00:00+0000" "+%s") + ;; + esac + echo writing source-date-epoch ... + echo "$timestamp" > "$SOURCE/source-date-epoch" +fi + +if test "$mpiname" = "mpich"; then + mpilicense="$SOURCE"/COPYRIGHT + otherlicenses=( + "$SOURCE"/modules/hwloc/COPYING + "$SOURCE"/modules/json-c/COPYING + "$SOURCE"/modules/yaksa/COPYRIGHT + ) +fi +if test "$mpiname" = "openmpi"; then + mpilicense="$SOURCE"/LICENSE + otherlicenses=( + "$SOURCE"/3rd-party/hwloc-*/COPYING + "$SOURCE"/3rd-party/libevent-*/LICENSE + "$SOURCE"/3rd-party/openpmix/LICENSE + "$SOURCE"/3rd-party/prrte/LICENSE + "$SOURCE"/3rd-party/treematch/COPYING + "$SOURCE"/opal/mca/event/libevent2022/libevent/LICENSE + "$SOURCE"/opal/mca/pmix/pmix3x/pmix/LICENSE + "$SOURCE"/ompi/mca/topo/treematch/treematch/COPYING + ) +fi +echo copying MPI license file... +cp "$mpilicense" "$PACKAGE/LICENSE" +if test -n "${ofidestdir+x}"; then + echo copying OFI license file... + cp "$ofidestdir/COPYING" "$PACKAGE/LICENSE.ofi" +fi +if test -n "${ucxdestdir+x}"; then + echo copying UCX license file... + cp "$ucxdestdir/LICENSE" "$PACKAGE/LICENSE.ucx" fi -echo copying license file... -cp "$SOURCE"/"$license" "$PACKAGE/LICENSE" +for license in "${otherlicenses[@]}"; do + test -f "$license" || continue + module=$(basename "$(dirname "$license")") + module="${module%%-[0-9]*}" + echo copying "$module" license file... + cp "$license" "$PACKAGE/LICENSE.$module" +done diff --git a/cibw-build-mpi.sh b/cibw-build-mpi.sh index f68b325..eb25b15 100755 --- a/cibw-build-mpi.sh +++ b/cibw-build-mpi.sh @@ -1,47 +1,151 @@ #!/bin/bash set -euo pipefail -mpiname="${MPINAME:-mpich}" - PROJECT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE=$PROJECT/package SOURCE=${SOURCE:-$PACKAGE/source} WORKDIR=${WORKDIR:-$PACKAGE/workdir} DESTDIR=${DESTDIR:-$PACKAGE/install} + +test -f "$PACKAGE"/METADATA +mpiname=$(awk '/Name/{print $2}' "$PACKAGE"/METADATA) +version=$(awk '/Version/{print $2}' "$PACKAGE"/METADATA) + +case $(uname) in + Linux) njobs=$(nproc);; + Darwin) njobs=$(sysctl -n hw.physicalcpu);; +esac + +if test "$(uname)" = Darwin; then + if test -z "${MACOSX_DEPLOYMENT_TARGET+x}"; then + case $(uname -m) in + arm64) export MACOSX_DEPLOYMENT_TARGET=11.0 ;; + x86_64) export MACOSX_DEPLOYMENT_TARGET=10.15 ;; + esac + fi + if test -z "${ZERO_AR_DATE+x}"; then + export ZERO_AR_DATE=1 + fi +fi + PREFIX=${PREFIX:-"/opt/$mpiname"} +build_cflags+=("-ffile-prefix-map=$SOURCE=/$mpiname-$version") +build_cflags+=("-ffile-prefix-map=$WORKDIR=/$mpiname-$version") +build_ldflags=() +if test "$(uname)" = Linux; then + build_cflags+=("-Wl,--build-id=none") + build_ldflags+=("-Wl,--as-needed") +fi +if test "$(uname)" = Darwin; then + build_ldflags+=("-Wl,-dead_strip_dylibs") +fi + +case "$mpiname" in + mpich) MODSOURCE="$SOURCE"/modules ;; + openmpi) MODSOURCE="$SOURCE"/3rd-party ;; +esac + +if test -d "$MODSOURCE"/ucx-*; then + ucxdir=$(basename "$(ls -d "$MODSOURCE"/ucx-*)") + UCXSOURCE="$MODSOURCE"/"$ucxdir" + UCXWORKDIR="$WORKDIR"/"$(basename "$MODSOURCE")"/"$ucxdir" + mkdir -p "$UCXWORKDIR" && cd "$UCXWORKDIR" + if test ! -e "$UCXWORKDIR"/config.log; then + echo running configure for UCX + ucxconfigure="$UCXSOURCE"/contrib/configure-release-mt + ucxoptions=(--prefix="$PREFIX" --disable-static) + "$ucxconfigure" "${ucxoptions[@]}" \ + CFLAGS="${build_cflags[*]}" \ + LDFLAGS="${build_ldflags[*]}" + fi + echo running make with "${njobs:-1}" jobs for UCX + make -j "${njobs:-1}" install-strip DESTDIR="$DESTDIR" + for lib in "${DESTDIR}${PREFIX}"/lib/libuc[mpst].la; do + sed -i "/dependency_libs\s*=/s|\($PREFIX\)|$DESTDIR\1|g" "$lib" + done + for pkg in "${DESTDIR}${PREFIX}"/lib/pkgconfig/ucx*.pc; do + sed -i "/prefix\s*=/s|\($PREFIX\)|$DESTDIR\1|g" "$pkg" + done +fi + +if test -d "$MODSOURCE"/libfabric-*; then + ofidir=$(basename "$(ls -d "$MODSOURCE"/libfabric-*)") + OFISOURCE="$MODSOURCE"/"$ofidir" + OFIWORKDIR="$WORKDIR"/"$(basename "$MODSOURCE")"/"$ofidir" + mkdir -p "$OFIWORKDIR" && cd "$OFIWORKDIR" + if test ! -e "$OFIWORKDIR"/config.log; then + echo running configure for OFI + oficonfigure="$OFISOURCE"/configure + ofioptions=(--prefix="$PREFIX" --disable-static) + "$oficonfigure" "${ofioptions[@]}" \ + CFLAGS="${build_cflags[*]}" \ + LDFLAGS="${build_ldflags[*]}" + fi + echo running make with "${njobs:-1}" jobs for OFI + make -j "${njobs:-1}" install-strip DESTDIR="$DESTDIR" + for lib in "${DESTDIR}${PREFIX}"/lib/libfabric*.la; do + sed -i "/dependency_libs\s*=/s|\($PREFIX\)|$DESTDIR\1|g" "$lib" + done + for pkg in "${DESTDIR}${PREFIX}"/lib/pkgconfig/libfabric*.pc; do + sed -i "/prefix\s*=/s|\($PREFIX\)|$DESTDIR\1|g" "$pkg" + done +fi + if test "$mpiname" = "mpich"; then - version=$(sed -n 's/MPICH_VERSION=\(.*\)/\1/p' "$SOURCE"/maint/Version) + case $(uname) in + Linux) netmod=ucx,ofi ;; + Darwin) netmod=ofi ;; + esac options=( CC=cc CXX=c++ --prefix="$PREFIX" - --with-device=ch4:"${variant:-ofi}" + --with-device="ch4:$netmod" --with-pm=hydra:gforker - --with-libfabric=embedded --with-ucx=embedded + --with-libfabric=embedded --with-hwloc=embedded --with-yaksa=embedded --disable-cxx --disable-doc + --disable-debug + --disable-dlopen --disable-static + --disable-pci --disable-opencl --disable-libxml2 --disable-dependency-tracking ) + if test -d "$SOURCE"/modules/ucx-*; then + options=("${options[@]/--with-ucx=embedded/--with-ucx=$DESTDIR$PREFIX}") + fi + if test -d "$SOURCE"/modules/libfabric-*; then + options=("${options[@]/--with-libfabric=embedded/--with-libfabric=$DESTDIR$PREFIX}") + fi + generated_files+=(src/include/mpichinfo.h) + generated_files+=(src/pm/hydra/hydra_config.h) + generated_files+=(src/pm/hydra/include/hydra_config.h) + generated_files+=(modules/ucx/config.h) + generated_files+=(modules/ucx/src/tools/info/build_config.h) + export BASH_SHELL="/bin/bash" + export MPICHLIB_CFLAGS="${build_cflags[*]}" + export MPICHLIB_LDFLAGS="${build_ldflags[*]}" if test "${version%%.*}" -lt 4; then options=("${options[@]/--disable-cxx}") - export FCFLAGS=-fallow-argument-mismatch + options=("${options[@]}" --disable-numa) + export CFLAGS=$MPICHLIB_CFLAGS + export LDFLAGS=$MPICHLIB_LDFLAGS export FFLAGS=-fallow-argument-mismatch + export FCFLAGS=-fallow-argument-mismatch + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:$DESTDIR$PREFIX/lib" fi if test "$(uname)" = Darwin; then export MPICH_MPICC_LDFLAGS="-Wl,-rpath,$PREFIX/lib" export MPICH_MPICXX_LDFLAGS="-Wl,-rpath,$PREFIX/lib" export MPICH_MPIFORT_LDFLAGS="-Wl,-rpath,$PREFIX/lib" fi - disable_doc='s/^\(install-data-local:\s\+\)\$/\1#\$/' - sed -i.orig "$disable_doc" "$SOURCE"/Makefile.in fi if test "$mpiname" = "openmpi"; then @@ -49,55 +153,154 @@ if test "$mpiname" = "openmpi"; then CC=cc CXX=c++ --prefix="$PREFIX" - --disable-dlopen - --disable-oshmem - --without-ofi --without-ucx - --without-psm2 + --without-ofi --without-cuda --without-rocm + --with-hwloc=internal + --with-libevent=internal --with-pmix=internal --with-prrte=internal - --with-libevent=internal - --with-hwloc=internal + --enable-ipv6 + --disable-doc + --disable-debug + --disable-dlopen --disable-static + --disable-oshmem + --disable-libompitrace + --disable-pci --disable-opencl --disable-libxml2 - --disable-libompitrace --enable-mpi-fortran=mpifh --disable-dependency-tracking ) + if test -d "$SOURCE"/3rd-party/ucx-*; then + options=("${options[@]/--without-ucx/--with-ucx=$DESTDIR$PREFIX}") + fi + if test -d "$SOURCE"/3rd-party/libfabric-*; then + options=("${options[@]/--without-ofi/--with-ofi=$DESTDIR$PREFIX}") + fi + if test "${version%%.*}" -lt 5; then + options=("${options[@]/--disable-pci/--disable-hwloc-pci}") + fi + generated_files+=(ompi/tools/ompi_info/Makefile) + generated_files+=(oshmem/tools/oshmem_info/Makefile) + generated_files+=(3rd-party/openpmix/src/tools/*/Makefile) + generated_files+=(3rd-party/prrte/src/tools/*/Makefile) + generated_files+=(opal/include/opal_config.h) + generated_files+=(opal/include/opal/version.h) + generated_files+=(3rd-party/openpmix/src/include/pmix_config.h) + generated_files+=(3rd-party/prrte/src/include/prte_config.h) + generated_files+=(ompi/tools/mpisync/Makefile) + generated_files+=(orte/tools/orte-info/Makefile) + generated_files+=(opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/Makefile) + generated_files+=(opal/mca/pmix/pmix3x/pmix/src/include/pmix_config.h) + export CFLAGS="${build_cflags[*]}" + export LDFLAGS="${build_ldflags[*]}" + export USER=user HOSTNAME=localhost fi -if test "$(uname)" = Darwin; then - export MACOSX_DEPLOYMENT_TARGET="11.0" - if test "$(uname -m)" = x86_64; then - export MACOSX_DEPLOYMENT_TARGET="10.9" - export ac_cv_func_aligned_alloc="no" # macOS>=10.15 - fi +mkdir -p "$WORKDIR" && cd "$WORKDIR" + +if test ! -e "$WORKDIR"/config.log; then + echo running configure for MPI + "$SOURCE"/configure "${options[@]}" || cat config.log + # shellcheck disable=SC2206 + generated_files=(${generated_files[@]:-}) + for filename in "${generated_files[@]}"; do + test -n "$filename" || continue + test -f "$filename" || continue + source="s|$SOURCE|/$mpiname-$version|g" + workdir="s|$WORKDIR|/$mpiname-$version|g" + destdir="s|$DESTDIR||g" + echo removing source/build/install paths in "$filename" + if test "$(basename "$filename")" = "Makefile"; then + sed -i.orig "/-D.*_BUILD_CFLAGS=/$source" "$filename" + sed -i.orig "/-D.*_BUILD_CFLAGS=/$workdir" "$filename" + sed -i.orig "/-D.*_BUILD_CFLAGS=/$destdir" "$filename" + sed -i.orig "/-D.*_BUILD_CPPFLAGS=/$source" "$filename" + sed -i.orig "/-D.*_BUILD_CPPFLAGS=/$workdir" "$filename" + sed -i.orig "/-D.*_BUILD_CPPFLAGS=/$destdir" "$filename" + sed -i.orig "/-D.*_BUILD_LIBS=/$source" "$filename" + sed -i.orig "/-D.*_BUILD_LIBS=/$workdir" "$filename" + sed -i.orig "/-D.*_BUILD_LIBS=/$destdir" "$filename" + else + sed -i.orig "$source" "$filename" + sed -i.orig "$workdir" "$filename" + sed -i.orig "$destdir" "$filename" + fi + done fi -case $(uname) in - Linux) njobs=$(nproc);; - Darwin) njobs=$(sysctl -n hw.physicalcpu);; -esac +echo running make with "${njobs:-1}" jobs for MPI +make -j "${njobs:-1}" install-strip DESTDIR="$DESTDIR" + +fixup-ucx() { + +cd "${DESTDIR}${PREFIX}" +rm -fr include/uc[mpst] +rm -f bin/io_demo +rm -f bin/ucx_* +rm -fr etc/ucx +rm -f lib/libuc[mpst]*.a +rm -f lib/libuc[mpst]*.la +rm -f lib/ucx/libuc[mt]_*.a +rm -f lib/ucx/libuc[mt]_*.la +rm -fr lib/cmake/ucx +rm -f lib/pkgconfig/ucx*.pc +rm -fr share/ucx -mkdir -p "$WORKDIR" -cd "$WORKDIR" +cd "${DESTDIR}${PREFIX}/lib" +test -f libucp.so || return 0 +for lib in libuc[mpst]*.so.?; do + if test -f "$lib".*; then + mv "$(readlink "$lib")" "$lib" + ln -sf "$lib" "${lib%.*}" + fi + patchelf --set-rpath "\$ORIGIN" "$lib" +done +patchelf --add-rpath "\$ORIGIN/ucx" libucm.so.? +patchelf --add-rpath "\$ORIGIN/ucx" libuct.so.? +for lib in ucx/libuc[mt]_*.so.?; do + if test -f "$lib".*; then + mv "$(dirname "$lib")/$(readlink "$lib")" "$lib" + ln -srf "$lib" "${lib%.*}" + fi + patchelf --set-rpath "\$ORIGIN" "$lib" + patchelf --add-rpath "\$ORIGIN/.." "$lib" +done -echo running configure -"$SOURCE"/configure "${options[@]}" || cat config.log +} # fixup-ucx() -echo running make with "${njobs:-1}" jobs -make -j "${njobs:-1}" install DESTDIR="$DESTDIR" +fixup-ofi() { -fixup-mpich() { +cd "${DESTDIR}${PREFIX}" +rm -fr include/rdma +rm -f bin/fi_* +rm -f lib/libfabric.a +rm -f lib/libfabric.la +rm -f lib/pkgconfig/libfabric.pc +rm -f share/man/man?/fabric.? +rm -f share/man/man?/fi_*.? + +cd "${DESTDIR}${PREFIX}/lib" +test -f libfabric.so || return 0 +for lib in libfabric.so.?; do + if test -f "$lib".*.*; then + mv "$(readlink "$lib")" "$lib" + ln -sf "$lib" "${lib%.*}" + fi + patchelf --set-rpath "\$ORIGIN" "$lib" +done + +} # fixup-ofi() + +fixup-mpi-mpich() { cd "${DESTDIR}${PREFIX}" rm -f include/*cxx.h rm -f include/*.mod rm -f include/*f.h -rm -fr include/rdma rm -f bin/mpif77 rm -f bin/mpif90 rm -f bin/mpifort @@ -112,13 +315,6 @@ rm -f lib/lib*mpifort.* rm -fr lib/pkgconfig rm -fr share -cd "${DESTDIR}${PREFIX}" -rm -f bin/io_demo -rm -f bin/ucx_read_profile -rm -f lib/libuc[mpst]*.la -rm -f lib/ucx/libuct_*.la -rm -fr lib/cmake - headers=(mpi.h) scripts=(mpicc mpic++ mpicxx) executables=(mpichversion mpivars) @@ -135,7 +331,11 @@ for script in "${scripts[@]}"; do # shellcheck disable=SC2016 topdir='$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE:-$0}")/.." \&\& pwd)' sed -i.orig s@^prefix=.*@prefix="$topdir"@ "$script" + sed -i.orig s:"\s*${build_cflags[*]}"::g "$script" sed -i.orig s:"$PREFIX":\"\$\{prefix\}\":g "$script" + sed -i.orig s:"$WORKDIR.*/lib.*\.la"::g "$script" + sed -i.orig s:"$DESTDIR.*/lib.*\.la"::g "$script" + sed -i.orig s:"$DESTDIR"::g "$script" sed -i.orig s:-Wl,-commons,use_dylibs::g "$script" sed -i.orig s:/usr/bin/bash:/bin/bash:g "$script" sed -i.orig s:-lmpicxx::g "$script" @@ -143,48 +343,54 @@ for script in "${scripts[@]}"; do done if test "$(uname)" = Linux; then - libmpi="libmpi.so.12" cd "${DESTDIR}${PREFIX}/bin" for exe in "${executables[@]}"; do patchelf --set-rpath "\$ORIGIN/../lib" "$exe" done cd "${DESTDIR}${PREFIX}/lib" - if test -f "$libmpi".*.*; then - mv "$(readlink "$libmpi")" "$libmpi" - ln -sf "$libmpi" "${libmpi%.*}" + for lib in libfabric.so.? libuc[mpst]*.so.?; do + test -f "$lib" || continue + for exe in "${executables[@]}"; do + patchelf --remove-needed "$lib" "../bin/$exe" + done + done + cd "${DESTDIR}${PREFIX}/lib" + rm -fr "$mpiname" + if test -f libfabric.so; then + mkdir -p "$mpiname" + mv libfabric.* "$mpiname" fi if test -f libucp.so; then - patchelf --set-rpath "\$ORIGIN" "$libmpi" - for lib in libuc[mpst]*.so.?; do - if test -f "$lib".*; then - mv "$(readlink "$lib")" "$lib" - ln -sf "$lib" "${lib%.*}" - fi - patchelf --set-rpath "\$ORIGIN" "$lib" - for exe in "${executables[@]}"; do - patchelf --remove-needed "$lib" "../bin/$exe" - done - done - patchelf --add-rpath "\$ORIGIN/ucx" libuct.so.? - for lib in ucx/libuct_*.so.?; do - if test -f "$lib".*; then - mv "$(dirname "$lib")/$(readlink "$lib")" "$lib" - ln -srf "$lib" "${lib%.*}" - fi - patchelf --set-rpath "\$ORIGIN/.." "$lib" - done + mkdir -p "$mpiname" + mv libuc[mpst]*.* ucx "$mpiname" + fi + cd "${DESTDIR}${PREFIX}/lib" + for lib in lib*.so; do + patchelf --set-rpath "\$ORIGIN" "$lib" + soname=$(patchelf --print-soname "$lib") + if test -L "$soname"; then + mv "$(readlink "$soname")" "$soname" + ln -sf "$soname" "$lib" + fi + done + if test -d "$mpiname"; then + patchelf --add-rpath "\$ORIGIN/$mpiname" libmpi.so.* fi + cd "${DESTDIR}${PREFIX}/lib" + find . -name '*.so' -type l -delete + ln -s libmpi.so.* libmpi.so fi if test "$(uname)" = Darwin; then libdir="$PREFIX/lib" - libmpi="libmpi.12.dylib" - libpmpi="libpmpi.12.dylib" + libmpi=$(basename "${DESTDIR}$libdir"/libmpi.*.dylib) + libpmpi=$(basename "${DESTDIR}$libdir"/libpmpi.*.dylib) cd "${DESTDIR}${PREFIX}/bin" for exe in "${executables[@]}"; do - install_name_tool -change "$libdir/$libmpi" "@rpath/$libmpi" "$exe" - install_name_tool -change "$libdir/$libpmpi" "@rpath/$libpmpi" "$exe" install_name_tool -add_rpath "@executable_path/../lib/" "$exe" + for lib in "$libmpi" "$libpmpi"; do + install_name_tool -change "$libdir/$lib" "@rpath/$lib" "$exe" + done done cd "${DESTDIR}${PREFIX}/lib" for lib in "$libmpi" "$libpmpi"; do @@ -192,11 +398,15 @@ if test "$(uname)" = Darwin; then install_name_tool -add_rpath "@loader_path/" "$lib" done install_name_tool -change "$libdir/$libpmpi" "@rpath/$libpmpi" "$libmpi" + cd "${DESTDIR}${PREFIX}/lib" + find . -name '*.dylib' -type l -delete + ln -s "$libmpi" libmpi.dylib + ln -s "$libpmpi" libpmpi.dylib fi -} # fixup-mpich() +} # fixup-mpi-mpich() -fixup-openmpi() { +fixup-mpi-openmpi() { cd "${DESTDIR}${PREFIX}" rm -fr include/ev* @@ -227,6 +437,7 @@ rm -f bin/psched rm -f bin/pterm rm -fr sbin +rm -f bin/pcc rm -f bin/pmixcc rm -f bin/mpif77 rm -f bin/mpif90 @@ -235,9 +446,12 @@ rm -f bin/oshrun rm -f lib/*.mod rm -f lib/lib*.a rm -f lib/lib*.la +rm -f lib/*/lib*.a +rm -f lib/*/lib*.la rm -f lib/libmpi_mpifh*.* rm -f lib/libmpi_usempi*.* rm -f lib/libompitrace.* +rm -fr lib/cmake rm -fr lib/openmpi rm -fr lib/pkgconfig rm -fr share/bash-completion @@ -245,6 +459,7 @@ rm -fr share/doc rm -fr share/man rm -fr share/hwloc rm -fr share/prte/rst +rm -fr share/ucx rm -f share/openmpi/mpif77-wrapper-data.txt rm -f share/openmpi/mpif90-wrapper-data.txt rm -f share/openmpi/mpifort-wrapper-data.txt @@ -252,6 +467,10 @@ rm -f share/pmix/pmixcc-wrapper-data.txt rm -f share/*/*.supp rm -f share/*/*/example.conf +cd "${DESTDIR}${PREFIX}/bin" +rm -f ompirun_wrapper +rm -f orterun_wrapper + cd "${DESTDIR}${PREFIX}/bin" unset executables for exe in 'mpirun' 'ompi*' 'pmix*' 'prte*' 'opal*' 'orte*'; do @@ -274,6 +493,22 @@ if test "$(uname)" = Linux; then patchelf --set-rpath "\$ORIGIN/../lib" "$exe" done cd "${DESTDIR}${PREFIX}/lib" + for lib in libfabric.so.? libuc[mpst]*.so.?; do + test -f "$lib" || continue + for exe in "${executables[@]}"; do + patchelf --remove-needed "$lib" "../bin/$exe" + done + done + cd "${DESTDIR}${PREFIX}/lib" + rm -fr "$mpiname" + if test -f libfabric.so; then + mkdir -p "$mpiname" + mv libfabric.* "$mpiname" + fi + if test -f libucp.so; then + mkdir -p "$mpiname" + mv libuc[mpst]*.* ucx "$mpiname" + fi for lib in lib*.so; do patchelf --set-rpath "\$ORIGIN" "$lib" soname=$(patchelf --print-soname "$lib") @@ -282,6 +517,33 @@ if test "$(uname)" = Linux; then ln -sf "$soname" "$lib" fi done + cd "${DESTDIR}${PREFIX}/lib" + if true; then + dependencies=( + libevent*.so.* + libhwloc.so.* + libpmix.so.* + libprrte.so.* + ) + for lib in "${dependencies[@]}"; do + test -f "$lib" || continue + mkdir -p "$mpiname" + mv "$lib" "$mpiname" + done + fi + cd "${DESTDIR}${PREFIX}/lib" + if test -d "$mpiname"; then + exepath="\$ORIGIN/../lib/$mpiname" + libpath="\$ORIGIN/$mpiname" + for exe in "${executables[@]}"; do + patchelf --add-rpath "$exepath" ../bin/"$exe" + done + patchelf --add-rpath "$libpath" libmpi.so.* + patchelf --add-rpath "$libpath" libopen-*.so.* + fi + cd "${DESTDIR}${PREFIX}/lib" + find . -name '*.so' -type l -delete + ln -s libmpi.so.* libmpi.so fi if test "$(uname)" = Darwin; then @@ -303,6 +565,7 @@ if test "$(uname)" = Darwin; then for lib in lib*.*.dylib; do install_name_tool -id "@rpath/$lib" "$lib" install_name_tool -add_rpath "@loader_path/" "$lib" + unset dependencies while IFS= read -r dep do dependencies+=("$dep") done < <(otool -L "$lib" | awk '(NR>1) {print $1}') @@ -313,6 +576,33 @@ if test "$(uname)" = Darwin; then fi done done + cd "${DESTDIR}${PREFIX}/lib" + if true; then + dependencies=( + libevent*.*.dylib + libhwloc.*.dylib + libpmix.*.dylib + libprrte.*.dylib + ) + for lib in "${dependencies[@]}"; do + test -f "$lib" || continue + mkdir -p "$mpiname" + mv "$lib" "$mpiname" + done + fi + cd "${DESTDIR}${PREFIX}/lib" + if test -d "$mpiname"; then + exepath="@executable_path/../lib/$mpiname/" + libpath="@loader_path/$mpiname/" + for exe in "${executables[@]}"; do + install_name_tool -add_rpath "$exepath" ../bin/"$exe" + done + install_name_tool -add_rpath "$libpath" libmpi.*.dylib + install_name_tool -add_rpath "$libpath" libopen-*.*.dylib + fi + cd "${DESTDIR}${PREFIX}/lib" + find . -name '*.dylib' -type l -delete + ln -s libmpi.*.dylib libmpi.dylib fi cd "${DESTDIR}${PREFIX}/share/openmpi" @@ -326,6 +616,8 @@ for cmd in mpicc mpic++ mpicxx mpiCC ortecc; do flags="\(linker_flags=-L\${libdir}\) *" rpath="-Wl,-rpath,\${libdir}" sed -i.orig "s:^$flags$:\1 $rpath:" "$wrapper_data" + libdir="$DESTDIR$PREFIX/lib" + sed -i.orig "s:$libdir:\${libdir}:g" "$wrapper_data" rm "$wrapper_data".orig done @@ -333,7 +625,7 @@ cd "${DESTDIR}${PREFIX}/bin" wrapper_cmd=opal_wrapper wrapper_src="$PROJECT/cibw-ompi-wrapper.c" wrapper_bin="$WORKDIR/cibw-ompi-wrapper.exe" -cc -DWRAPPER="$wrapper_cmd" "$wrapper_src" -o "$wrapper_bin" +cc -O -DWRAPPER="$wrapper_cmd" "$wrapper_src" -o "$wrapper_bin" executables=(mpicc mpic++ mpicxx mpiCC ortecc) for exe in "${executables[@]}"; do test -e "$exe" || continue @@ -353,15 +645,30 @@ mv "$executor" "$wrapper" ln -s "$wrapper" "$executor" wrapper_src="$PROJECT/cibw-ompi-wrapper.c" wrapper_bin="$WORKDIR/cibw-ompi-wrapper.exe" -cc -DWRAPPER="$wrapper" "$wrapper_src" -o "$wrapper_bin" +cc -O -DWRAPPER="$wrapper" "$wrapper_src" -o "$wrapper_bin" executables=(mpirun mpiexec orterun) for exe in "${executables[@]}"; do test -e "$exe" || continue test -L "$exe" || continue - install "$wrapper_bin" "$exe" + install -s "$wrapper_bin" "$exe" done -} # fixup-openmpi() +} # fixup-mpi-openmpi() -echo fixing install tree -fixup-"$mpiname" +echo fixing UCX install tree +fixup-ucx +echo fixing OFI install tree +fixup-ofi +echo fixing MPI install tree +fixup-mpi-"$mpiname" + +echo checking install tree +cd "${DESTDIR}${PREFIX}" +dirty=0 +echo checking for files with SOURCE +if grep -lr "$SOURCE" ; then dirty=1; fi +echo checking for files with WORKDIR +if grep -lr "$WORKDIR"; then dirty=1; fi +echo checking for files with DESTDIR +if grep -lr "$DESTDIR"; then dirty=1; fi +test "$dirty" -eq 0 diff --git a/cibw-check-mpi.sh b/cibw-check-mpi.sh index a7306e6..d1436ed 100755 --- a/cibw-check-mpi.sh +++ b/cibw-check-mpi.sh @@ -20,7 +20,11 @@ int main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Get_processor_name(name, &len); + if (rank != 0) + MPI_Recv(name, 0 , MPI_BYTE, rank-1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); printf("Hello, World! I am process %d of %d on %s.\n", rank, size, name); + if (rank != size - 1) + MPI_Send(name, 0 , MPI_BYTE, rank+1, 0, MPI_COMM_WORLD); MPI_Finalize(); return 0; } @@ -56,6 +60,14 @@ fi if test "$mpiname" = "openmpi"; then RUN command -v ompi_info RUN ompi_info + if command -v pmix_info > /dev/null; then + RUN command -v pmix_info + RUN pmix_info + fi + if command -v prte_info > /dev/null; then + RUN command -v prte_info + RUN prte_info + fi fi RUN command -v mpicc @@ -69,3 +81,23 @@ RUN mpicxx helloworld.cxx -o helloworld-cxx RUN command -v mpiexec RUN mpiexec -n 3 ./helloworld-c RUN mpiexec -n 3 ./helloworld-cxx + +if test "$mpiname" = "mpich"; then + version=$(mpichversion --version | cut -d':' -f 2) + case $(uname) in + Linux) ch4netmods=(ofi ucx) ;; + Darwin) ch4netmods=(ofi) ;; + esac + export MPICH_CH4_UCX_CAPABILITY_DEBUG=1 + export MPICH_CH4_OFI_CAPABILITY_DEBUG=1 + for netmod in "${ch4netmods[@]}"; do + printf "testing ch4:%s ... " "$netmod" + export MPICH_CH4_NETMOD="$netmod" + test "${version%%.*}" -ge 4 || netmod="" + ./helloworld-c | grep -i "$netmod" > /dev/null + printf "OK\n" + done + unset MPICH_CH4_UCX_CAPABILITY_DEBUG + unset MPICH_CH4_OFI_CAPABILITY_DEBUG + unset MPICH_CH4_NETMOD +fi diff --git a/cibw-ompi-wrapper.c b/cibw-ompi-wrapper.c index c967ebc..16e72dc 100644 --- a/cibw-ompi-wrapper.c +++ b/cibw-ompi-wrapper.c @@ -34,7 +34,7 @@ int main(int argc, char *argv[]) { (void) readlink("/proc/curproc/file", exe, PATH_MAX); #elif defined(__linux__) (void) readlink("/proc/self/exe", exe, PATH_MAX); -#elif defined( __sun) +#elif defined(__sun) (void) strncpy(exe, getexecname(), PATH_MAX); #else # error unknown system diff --git a/cibw-patch-cmd.sh b/cibw-patch-cmd.sh new file mode 100755 index 0000000..21aa773 --- /dev/null +++ b/cibw-patch-cmd.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euo pipefail + +case "$(uname)" in + Linux) toolname=auditwheel;; + Darwin) toolname=delocate-wheel;; +esac + +topdir=$(cd "$(dirname -- "$0")" && pwd -P) +toolpatch="$topdir/patches/$toolname.py" +test -f "$toolpatch" || exit 0 + +filename=$(command -v "$toolname") +shebang=$(head -n 1 "$filename") +sed "1 s|^.*$|$shebang|" "$toolpatch" > "$filename" diff --git a/package/pyproject.toml b/package/pyproject.toml index 9787c3b..ac76de3 100644 --- a/package/pyproject.toml +++ b/package/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools", "wheel"] +requires = ["setuptools >= 70.1"] build-backend = "setuptools.build_meta" diff --git a/package/setup.py b/package/setup.py index b515d00..0cd44ee 100644 --- a/package/setup.py +++ b/package/setup.py @@ -1,5 +1,8 @@ from setuptools import setup -from wheel.bdist_wheel import bdist_wheel +try: + from setuptools.command.bdist_wheel import bdist_wheel +except ImportError: + from wheel.bdist_wheel import bdist_wheel import re import os @@ -9,22 +12,26 @@ class bdist_wheel(bdist_wheel): def finalize_options(self): super().finalize_options() self.root_is_pure = False - self.build_number = release or None def get_tag(self): plat_tag = super().get_tag()[-1] return (self.python_tag, "none", plat_tag) -mpiname = os.environ.get("MPINAME", "mpich") +with open("METADATA") as fobj: + metadata = re.search(r""" + Name:\s*(?P.*)\n + Version:\s*(?P.*)\n + """, fobj.read(), re.VERBOSE).groupdict() + +mpiname = metadata["name"] +version = metadata["version"] release = os.environ.get("RELEASE", "") -pkgname = mpiname +if release: + version += f".post{release}" if mpiname == "mpich": project = "MPICH" - version_re = re.compile(r""" - \#define\s+MPICH_VERSION\s+\"(.*)\" - """, re.VERBOSE) license = "LicenseRef-MPICH" author = "MPICH Team" author_email = "discuss@mpich.org" @@ -39,11 +46,6 @@ def get_tag(self): if mpiname == "openmpi": project = "Open MPI" - version_re = re.compile(r""" - \#define\s+OMPI_MAJOR_VERSION\s+(\d+)\n - \#define\s+OMPI_MINOR_VERSION\s+(\d+)\n - \#define\s+OMPI_RELEASE_VERSION\s+(\d+)\n - """, re.VERBOSE) license = "LicenseRef-OpenMPI" author = "Open MPI Team" author_email = "users@lists.open-mpi.org" @@ -57,18 +59,16 @@ def get_tag(self): } description = "A high performance implementation of MPI" -long_description = f"""`{project} <{project_urls["Homepage"]}>`_ is a -high-performance and widely portable implementation of the Message -Passing Interface (`MPI `_) standard.""" +long_description = f"""\ +`{project} <{project_urls["Homepage"]}>`_ \ +is a high-performance implementation of the +Message Passing Interface (`MPI `_) standard.""" prefix = os.environ.get("PREFIX", f"/opt/{mpiname}") basedir = os.path.dirname(__file__) destdir = os.environ.get("DESTDIR", f"{basedir}/install") rootdir = f"{destdir}{prefix}" -with open(f"{rootdir}/include/mpi.h") as fobj: - version = ".".join(version_re.search(fobj.read()).groups()) - data_files = [] for path, dirs, files in os.walk(rootdir): dirs.sort() @@ -82,10 +82,10 @@ def get_tag(self): } setup( - name=pkgname, + name=mpiname, version=version, license=license, - license_files=["LICENSE"], + license_files=["LICENSE", "LICENSE.*"], description=description, long_description=long_description, long_description_content_type="text/x-rst", diff --git a/patches/auditwheel.py b/patches/auditwheel.py new file mode 100644 index 0000000..01a0dce --- /dev/null +++ b/patches/auditwheel.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +# +# https://github.com/pypa/auditwheel/pull/517 +# + +import os +import sys +import auditwheel.main +import auditwheel.tools + +os_walk = os.walk + + +def walk(topdir, *args, **kwargs): + topdir = os.path.normpath(topdir) + for dirpath, dirnames, filenames in os_walk(topdir, *args, **kwargs): + # sort list of dirnames in-place such that `os.walk` + # will recurse into subdirectories in reproducible order + dirnames.sort() + # recurse into any top-level .dist-info subdirectory last + if dirpath == topdir: + subdirs = [] + dist_info = [] + for dir in dirnames: + if dir.endswith(".dist-info"): + dist_info.append(dir) + else: + subdirs.append(dir) + dirnames[:] = subdirs + dirnames.extend(dist_info) + del dist_info + # sort list of filenames for iteration in reproducible order + filenames.sort() + # list any dist-info/RECORD file last + if dirpath.endswith(".dist-info") and os.path.dirname(dirpath) == topdir: + if "RECORD" in filenames: + filenames.remove("RECORD") + filenames.append("RECORD") + yield dirpath, dirnames, filenames + + +def dir2zip(in_dir, zip_fname, date_time=None): + import zipfile + from datetime import datetime, timezone + in_dir = os.path.normpath(in_dir) + if date_time is None: + st = os.stat(in_dir) + date_time = datetime.fromtimestamp(st.st_mtime, tz=timezone.utc) + date_time_args = date_time.timetuple()[:6] + compression = zipfile.ZIP_DEFLATED + with zipfile.ZipFile(zip_fname, "w", compression=compression) as z: + for root, dirs, files in walk(in_dir): + if root != in_dir and not (dirs or files): + dname = root + out_dname = os.path.relpath(dname, in_dir) + "/" + zinfo = zipfile.ZipInfo.from_file(dname, out_dname) + zinfo.date_time = date_time_args + z.writestr(zinfo, b"") + for file in files: + fname = os.path.join(root, file) + out_fname = os.path.relpath(fname, in_dir) + zinfo = zipfile.ZipInfo.from_file(fname, out_fname) + zinfo.date_time = date_time_args + zinfo.compress_type = compression + with open(fname, "rb") as fp: + z.writestr(zinfo, fp.read()) + + +os.walk = walk +auditwheel.tools.dir2zip = dir2zip + +if __name__ == "__main__": + sys.exit(auditwheel.main.main()) diff --git a/patches/delocate-wheel.py b/patches/delocate-wheel.py new file mode 100644 index 0000000..6cd1588 --- /dev/null +++ b/patches/delocate-wheel.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +import sys +from delocate.cmd.delocate_wheel import main + +sys.argv[1:1] = """ +--ignore-missing-dependencies +""".split() + +if __name__ == "__main__": + sys.exit(main()) diff --git a/patches/mpich-4.2.3 b/patches/mpich-4.2.3 new file mode 100644 index 0000000..7abaa41 --- /dev/null +++ b/patches/mpich-4.2.3 @@ -0,0 +1,116 @@ +From 135459663554992589868b9f4e75d040e0cf4942 Mon Sep 17 00:00:00 2001 +From: Ken Raffenetti +Date: Wed, 23 Oct 2024 09:58:33 -0500 +Subject: [PATCH] ch4/ucx: Reorder function definitions + +When building a non-inline netmod configuration, the compiler will flag +MPIDI_UCX_do_am_recv for implicit declaration. Move its definition +before its used to resolve. Fixes pmodels/mpich#7185. +--- + src/mpid/ch4/netmod/ucx/ucx_am.c | 86 ++++++++++++++++---------------- + 1 file changed, 43 insertions(+), 43 deletions(-) + +diff --git a/src/mpid/ch4/netmod/ucx/ucx_am.c b/src/mpid/ch4/netmod/ucx/ucx_am.c +index 082d35f9cc8..05f849eac6d 100644 +--- a/src/mpid/ch4/netmod/ucx/ucx_am.c ++++ b/src/mpid/ch4/netmod/ucx/ucx_am.c +@@ -70,6 +70,49 @@ void MPIDI_UCX_am_send_callback(void *request, ucs_status_t status) + } + + #ifdef HAVE_UCP_AM_NBX ++/* Called when recv buffer is posted */ ++int MPIDI_UCX_do_am_recv(MPIR_Request * rreq) ++{ ++ void *recv_buf; ++ bool is_contig; ++ MPI_Aint data_sz, in_data_sz; ++ int vci = MPIDI_Request_get_vci(rreq); ++ ++ MPIDIG_get_recv_buffer(&recv_buf, &data_sz, &is_contig, &in_data_sz, rreq); ++ if (!is_contig || in_data_sz > data_sz) { ++ /* non-contig datatype, need receive into pack buffer */ ++ /* ucx will error out if buffer size is less than the promised data size, ++ * also use a pack buffer in this case */ ++ recv_buf = MPL_malloc(in_data_sz, MPL_MEM_OTHER); ++ MPIR_Assert(recv_buf); ++ MPIDI_UCX_AM_RECV_REQUEST(rreq, pack_buffer) = recv_buf; ++ } else { ++ MPIDI_UCX_AM_RECV_REQUEST(rreq, pack_buffer) = NULL; ++ } ++ ++ MPIDI_UCX_ucp_request_t *ucp_request; ++ size_t received_length; ++ ucp_request_param_t param = { ++ .op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_RECV_INFO, ++ .cb.recv_am = &MPIDI_UCX_am_recv_callback_nbx, ++ .recv_info.length = &received_length, ++ }; ++ void *data_desc = MPIDI_UCX_AM_RECV_REQUEST(rreq, data_desc); ++ /* note: use in_data_sz to match promised data size */ ++ ucp_request = ucp_am_recv_data_nbx(MPIDI_UCX_global.ctx[vci].worker, ++ data_desc, recv_buf, in_data_sz, ¶m); ++ if (ucp_request == NULL) { ++ /* completed immediately */ ++ MPIDI_UCX_ucp_request_t tmp_ucp_request; ++ tmp_ucp_request.req = rreq; ++ MPIDI_UCX_am_recv_callback_nbx(&tmp_ucp_request, UCS_OK, received_length, NULL); ++ } else { ++ ucp_request->req = rreq; ++ } ++ ++ return MPI_SUCCESS; ++} ++ + /* Am handler for messages sent from ucp_am_send_nbx. Registered with + * ucp_worker_set_am_recv_handler. + */ +@@ -116,49 +159,6 @@ ucs_status_t MPIDI_UCX_am_nbx_handler(void *arg, const void *header, size_t head + } + } + +-/* Called when recv buffer is posted */ +-int MPIDI_UCX_do_am_recv(MPIR_Request * rreq) +-{ +- void *recv_buf; +- bool is_contig; +- MPI_Aint data_sz, in_data_sz; +- int vci = MPIDI_Request_get_vci(rreq); +- +- MPIDIG_get_recv_buffer(&recv_buf, &data_sz, &is_contig, &in_data_sz, rreq); +- if (!is_contig || in_data_sz > data_sz) { +- /* non-contig datatype, need receive into pack buffer */ +- /* ucx will error out if buffer size is less than the promised data size, +- * also use a pack buffer in this case */ +- recv_buf = MPL_malloc(in_data_sz, MPL_MEM_OTHER); +- MPIR_Assert(recv_buf); +- MPIDI_UCX_AM_RECV_REQUEST(rreq, pack_buffer) = recv_buf; +- } else { +- MPIDI_UCX_AM_RECV_REQUEST(rreq, pack_buffer) = NULL; +- } +- +- MPIDI_UCX_ucp_request_t *ucp_request; +- size_t received_length; +- ucp_request_param_t param = { +- .op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_RECV_INFO, +- .cb.recv_am = &MPIDI_UCX_am_recv_callback_nbx, +- .recv_info.length = &received_length, +- }; +- void *data_desc = MPIDI_UCX_AM_RECV_REQUEST(rreq, data_desc); +- /* note: use in_data_sz to match promised data size */ +- ucp_request = ucp_am_recv_data_nbx(MPIDI_UCX_global.ctx[vci].worker, +- data_desc, recv_buf, in_data_sz, ¶m); +- if (ucp_request == NULL) { +- /* completed immediately */ +- MPIDI_UCX_ucp_request_t tmp_ucp_request; +- tmp_ucp_request.req = rreq; +- MPIDI_UCX_am_recv_callback_nbx(&tmp_ucp_request, UCS_OK, received_length, NULL); +- } else { +- ucp_request->req = rreq; +- } +- +- return MPI_SUCCESS; +-} +- + /* callback for ucp_am_recv_data_nbx */ + void MPIDI_UCX_am_recv_callback_nbx(void *request, ucs_status_t status, size_t length, + void *user_data) diff --git a/build-wheel.sh b/wheel-build.sh similarity index 60% rename from build-wheel.sh rename to wheel-build.sh index 89d5c44..299098e 100755 --- a/build-wheel.sh +++ b/wheel-build.sh @@ -6,32 +6,42 @@ WORKDIR=package/workdir DESTDIR=package/install ARCHLIST=${ARCHLIST:-$(uname -m)} -export CIBW_BUILD_FRONTEND='build' +read -r SOURCE_DATE_EPOCH < "$SOURCE"/source-date-epoch +export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH + +export CIBW_BUILD_FRONTEND='build[uv]' export CIBW_BUILD='cp313-*' export CIBW_SKIP='*musllinux*' export CIBW_ARCHS=$ARCHLIST export CIBW_BEFORE_ALL='bash {project}/cibw-build-mpi.sh' +export CIBW_BEFORE_BUILD='bash {project}/cibw-patch-cmd.sh' export CIBW_TEST_COMMAND='bash {project}/cibw-check-mpi.sh' export CIBW_ENVIRONMENT_PASS='MPINAME RELEASE SOURCE WORKDIR DESTDIR' -export CIBW_REPAIR_WHEEL_COMMAND_MACOS='delocate-wheel --ignore-missing-dependencies --exclude libmpi --exclude libpmpi --require-archs {delocate_archs -w {dest_dir} -v {wheel}' +export CIBW_MANYLINUX_AARCH64_IMAGE=manylinux_2_28 +export CIBW_MANYLINUX_X86_64_IMAGE=manylinux_2_28 if test "$(uname)" = Linux; then containerengine=$(basename "$(command -v podman || command -v docker)") export CIBW_CONTAINER_ENGINE=$containerengine export SOURCE="/project/$SOURCE" - export WORKDIR="/host/$PWD/$WORKDIR" - export DESTDIR="/host/$PWD/$DESTDIR" + export WORKDIR="/host$PWD/$WORKDIR" + export DESTDIR="/host$PWD/$DESTDIR" platform=linux fi if test "$(uname)" = Darwin; then + case "$(uname -m)" in + arm64) export MACOSX_DEPLOYMENT_TARGET=11.0 ;; + x86_64) export MACOSX_DEPLOYMENT_TARGET=10.15 ;; + esac + export CIBW_BUILD_FRONTEND='build' + export CIBW_BUILD='pp310-*' export SOURCE="$PWD/$SOURCE" export WORKDIR="$PWD/$WORKDIR" export DESTDIR="$PWD/$DESTDIR" - export CIBW_BUILD='pp310-*' platform=macos fi -python -m pipx run \ +pipx run \ cibuildwheel \ --platform "$platform" \ --output-dir "${1:-dist}" \ diff --git a/check-wheel.sh b/wheel-check.sh similarity index 70% rename from check-wheel.sh rename to wheel-check.sh index 0b7e582..6db8f46 100755 --- a/check-wheel.sh +++ b/wheel-check.sh @@ -16,7 +16,9 @@ unzip -qq "$wheelfile" -d "$workdir" cd "$workdir" whlname=$(basename "$wheelfile") -pkgname=${whlname%%-*} +whlinfo=${whlname%%-py*} +pkgname=${whlinfo%%-*} +version=${whlinfo##*-} mpiname=${pkgname} data=$(ls -d "$pkgname"-*.data/data) @@ -29,9 +31,12 @@ if test "$(uname)" = Linux; then libsdir=.libs print-runpath() { patchelf --print-rpath "$1"; } print-needed() { patchelf --print-needed "$1"; } - if test -f "$data"/lib/libucp.so; then + if test -f "$data"/lib/*/libucp.so.*; then runlibs=$runlibs'|libuc(m|p|s|t)'$soregex fi + if test -f "$data"/lib/*/libfabric.so.*; then + runlibs=$runlibs'|libfabric'$soregex + fi fi if test "$(uname)" = Darwin; then runpath='@executable_path/../lib/|@loader_path/' @@ -41,9 +46,12 @@ if test "$(uname)" = Darwin; then libsdir=.dylibs print-runpath() { otool -l "$1" | sed -n '/RPATH/{n;n;p;}'; } print-needed() { otool -L "$1" | sed 1,1d; } - if test -f "$data"/lib/libucp.dylib; then + if test -f "$data"/lib/libucp.*.dylib; then runlibs=$runlibs'|libuc(m|p|s|t)'$soregex fi + if test -f "$data"/lib/libfabric.*.dylib; then + runlibs=$runlibs'|libfabric'$soregex + fi fi if test "$mpiname" = "mpich"; then @@ -65,12 +73,17 @@ if test "$mpiname" = "mpich"; then "$data"/bin/hydra_* ) libraries=( - "$data"/lib/libmpi.* + "$data"/lib/lib*mpi.* ) - if test -d "$data"/lib/ucx; then + if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then + libraries+=( + "$data"/lib/*/libfabric.* + ) + fi + if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then libraries+=( - "$data"/lib/libuc[mpst]*.* - "$data"/lib/ucx/libuc*.* + "$data"/lib/*/libuc[mpst]*.* + "$data"/lib/*/ucx/libuc[mpst]*.* ) fi fi @@ -96,6 +109,25 @@ if test "$mpiname" = "openmpi"; then "$data"/lib/libmpi.* "$data"/lib/libopen-*.* ) + if test "${version%%.*}" -ge 5; then + libraries+=( + "$data"/lib/openmpi/libevent*.* + "$data"/lib/openmpi/libhwloc.* + "$data"/lib/openmpi/libpmix.* + "$data"/lib/openmpi/libprrte.* + ) + fi + if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then + libraries+=( + "$data"/lib/*/libuc[mpst]*.* + "$data"/lib/*/ucx/libuc[mpst]*.* + ) + fi + if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then + libraries+=( + "$data"/lib/*/libfabric.* + ) + fi runlibs+='|lib(z|util|event.*|hwloc)'$soregex runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex fi @@ -103,18 +135,19 @@ fi check-binary() { local dso=$1 out1="" out2="" echo checking "$dso"... - test -f "$dso" || printf "ERROR: file not found\n" + test -f "$dso" || (printf "ERROR: file not found\n"; exit 1) out1="$(print-runpath "$dso" | grep -vE "$runpath" || true)" test -z "$out1" || printf "ERROR: RUNPATH\n%s\n" "$out1" out2="$(print-needed "$dso" | grep -vE "$runlibs" || true)" test -z "$out2" || printf "ERROR: NEEDED\n%s\n" "$out2" - test -z "$out1" -a -z "$out2" + test -z "$out1" + test -z "$out2" } for header in "${headers[@]-}"; do test -n "$header" || break echo checking "$header"... - test -f "$header" + test -f "$header" || (printf "ERROR: file not found\n"; exit 1) out=$(grep -E '^#include\s+"mpicxx\.h"' "$header" || true) test -z "$out" || printf "ERROR: include\n%s\n" "$out" test -z "$out" @@ -122,7 +155,7 @@ done for script in "${scripts[@]-}"; do test -n "$script" || break echo checking "$script"... - test -f "$script" + test -f "$script" || (printf "ERROR: file not found\n"; exit 1) out=$(grep -E "/opt/$mpiname" "$script" || true) test -z "$out" || printf "ERROR: prefix\n%s\n" "$out" test -z "$out" diff --git a/wheel-test.sh b/wheel-test.sh new file mode 100755 index 0000000..3bc2621 --- /dev/null +++ b/wheel-test.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +wheelhouse="${1:-wheelhouse}" +ls -d "$wheelhouse" > /dev/null + +venv=$(mktemp -d) +trap 'rm -rf $venv' EXIT + +mpiname=${MPINAME:-mpich} +version=${VERSION:-} +release=${RELEASE:-} +mpispec="$mpiname" +test -n "$version" && mpispec+="==$version" +test -n "$release" && mpispec+=".post$release" + +RUN() { echo + "$@"; "$@"; } + +RUN python3 -m venv "$venv" +RUN source "$venv"/bin/activate + +RUN python -m pip install "$mpispec" \ + --no-index --find-links "${1:-wheelhouse}" + +RUN ./cibw-check-mpi.sh