Skip to content

Commit

Permalink
Support for OFI and UCX
Browse files Browse the repository at this point in the history
* MPICH: Configure --with-device=ch4:ucx,ofi
* Open MPI: Configure --with-ucx --with-ofi
* Remove support for ppc64le architecture
* Use manylinux 2.28 image
* Reproducible builds
* Strip binaries to reduce size
* Remove unneeded binaries
* Package various licenses
  • Loading branch information
dalcinl committed Jan 20, 2025
1 parent 9f634bf commit 1092a0a
Show file tree
Hide file tree
Showing 21 changed files with 973 additions and 160 deletions.
62 changes: 40 additions & 22 deletions .github/workflows/cd-wheel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ on: # yamllint disable-line rule:truthy
- all
- Linux
- Linux-aarch64
- Linux-ppc64le
- Linux-x86_64
- macOS
- macOS-arm64
Expand All @@ -76,11 +75,11 @@ jobs:
shell: python
name: 'setup build matrix'
run: |
# setup build matrix # "
keys = ("os", "arch", "runner")
rows = [
("Linux", "aarch64", "ubuntu-22.04"),
("Linux", "ppc64le", "ubuntu-22.04"),
("Linux", "x86_64", "ubuntu-22.04"),
("Linux", "aarch64", "ubuntu-24.04-arm"),
("Linux", "x86_64", "ubuntu-24.04"),
("macOS", "arm64", "macos-14"),
("macOS", "x86_64", "macos-13"),
]
Expand All @@ -93,6 +92,7 @@ jobs:
import os, json
with open(os.getenv("GITHUB_OUTPUT"), "w") as out:
print(f"matrix={json.dumps(matrix)}", file=out)
# "
build:
needs: setup
Expand All @@ -113,16 +113,21 @@ jobs:
if: ${{ runner.os == 'macOS' }}
name: setup-macOS
run: |
# set macOS deployment target
case $(uname -m) in
arm64) echo MACOSX_DEPLOYMENT_TARGET=11.0 >> $GITHUB_ENV ;;
x86_64) echo MACOSX_DEPLOYMENT_TARGET=10.15 >> $GITHUB_ENV ;;
esac
# create gfortran symlink
cd $(brew --prefix)/bin
gfortran=$(ls gfortran-* | sort | head -n 1)
sudo ln -s $gfortran gfortran
# unlink libevent
brew unlink libevent || true
# install autotools
brew install autoconf
brew install automake
brew install libtool
# unlink libevent
brew unlink libevent || true
# install uv
brew install uv
Expand All @@ -131,20 +136,19 @@ jobs:
with:
python-version: 3

- id: setup-qemu
if: ${{ runner.os == 'Linux' }}
uses: docker/setup-qemu-action@v3
with:
platforms: all

- id: bootstrap
run: ./bootstrap.sh

- id: source-date-epoch
run: |
SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
read -r SOURCE_DATE_EPOCH < source-date-epoch || true
SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-$(git log -1 --pretty=%ct)}
echo SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH >> $GITHUB_ENV
echo $(git log -1 --pretty=%ci) [timestamp=$SOURCE_DATE_EPOCH]
test $(uname) = Darwin && (echo ZERO_AR_DATE=1 >> $GITHUB_ENV)
echo [SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH] $(
date -u -d @$SOURCE_DATE_EPOCH 2>/dev/null ||
date -u -r $SOURCE_DATE_EPOCH 2>/dev/null )
working-directory: package/source

- id: build
uses: pypa/[email protected]
Expand All @@ -159,6 +163,8 @@ jobs:
CIBW_ARCHS: "${{ matrix.arch }}"
CIBW_BEFORE_ALL: >-
bash {project}/cibw-build-mpi.sh
CIBW_BEFORE_BUILD: >-
bash {project}/cibw-patch-cmd.sh
CIBW_TEST_COMMAND: >-
bash {project}/cibw-check-mpi.sh
CIBW_ENVIRONMENT_PASS: >-
Expand All @@ -171,12 +177,8 @@ jobs:
SOURCE="$PWD/package/source"
WORKDIR="$PWD/package/workdir"
DESTDIR="$PWD/package/install"
CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
delocate-wheel
--ignore-missing-dependencies
--exclude libmpi --exclude libpmpi
--require-archs {delocate_archs}
-w {dest_dir} -v {wheel}
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28

- id: sha256sum
run: |
Expand All @@ -190,8 +192,24 @@ jobs:
- id: upload
uses: actions/upload-artifact@v4
with:
name: wheel-${{ inputs.mpiname }}-${{ matrix.os }}-${{ matrix.arch }}
name: "wheel-${{ inputs.mpiname }}-\
${{ inputs.version || 'latest' }}-\
${{ matrix.os }}-${{ matrix.arch }}"
path: wheelhouse/*.whl

- id: check
run: ./check-wheel.sh wheelhouse
run: ./wheel-check.sh wheelhouse

- id: test
run: |
# ./wheel-test.sh
test $(uname) = Linux && runner=(
docker run
-e MPINAME=${{ inputs.mpiname }}
-v $(pwd):/${{ github.workspace }}
-w ${{ github.workspace }}
--platform linux/${{
matrix.arch == 'aarch64' && 'arm64' ||
matrix.arch == 'x86_64' && 'amd64'
}} python:3)
${runner[@]:-} ./wheel-test.sh wheelhouse
1 change: 0 additions & 1 deletion .github/workflows/cd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ on: # yamllint disable-line rule:truthy
- all
- Linux
- Linux-aarch64
- Linux-ppc64le
- Linux-x86_64
- macOS
- macOS-arm64
Expand Down
24 changes: 22 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,32 @@ on: # yamllint disable-line rule:truthy

jobs:

mpich:
mpich-42:
uses: ./.github/workflows/cd-wheel.yml
with:
mpiname: mpich
version:

openmpi:
mpich-41:
uses: ./.github/workflows/cd-wheel.yml
with:
mpiname: mpich
version: 4.1.3

mpich-34:
uses: ./.github/workflows/cd-wheel.yml
with:
mpiname: mpich
version: 3.4.3

openmpi-50:
uses: ./.github/workflows/cd-wheel.yml
with:
mpiname: openmpi
version:

openmpi-41:
uses: ./.github/workflows/cd-wheel.yml
with:
mpiname: openmpi
version: 4.1.6
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@
/venv/
*.egg-info/
*.tar.gz
*.tar.bz2
*~
13 changes: 13 additions & 0 deletions .yamllint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# https://github.com/adrienverge/yamllint

extends: default
rules:
document-start: disable
colons:
max-spaces-before: 0
max-spaces-after: -1
indentation:
indent-sequences: whatever
line-length:
level: warning
max: 80
8 changes: 5 additions & 3 deletions LICENSE.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
BSD 2-Clause License

Copyright (c) 2024, MPI for Python
Copyright (c) 2025, Lisandro Dalcin

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Expand All @@ -12,6 +10,10 @@ modification, are permitted provided that the following conditions are met:
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
Expand Down
12 changes: 7 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,19 +1,21 @@
build:
./bootstrap.sh
./build-wheel.sh dist
./check-wheel.sh dist
./wheel-build.sh dist
./wheel-check.sh dist
./wheel-test.sh dist

lint:
codespell *.sh */*.py
shellcheck *.sh
ruff check -qn package/*.py
ruff check -qn */*.py
yamllint .github/

clean:
$(RM) -r package/METADATA
$(RM) -r package/LICENSE*
$(RM) -r package/build
$(RM) -r package/LICENSE
$(RM) -r package/install
$(RM) -r package/source
$(RM) -r package/workdir
$(RM) -r package/install
$(RM) -r package/*.egg-info
$(RM) -r .*_cache
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ This repository builds and publishes [MPICH](https://www.mpich.org) and
a variety of

- operating systems: *Linux*, *macOS*;
- processor architectures: *AMD64*, *ARM64*, *PPC64*;
- processor architectures: *AMD64*, *ARM64*;
- Python implementations: *CPython*, *PyPy*.

MPI wheels are uploaded to the [Anaconda.org](https://anaconda.org/mpi4py)
Expand Down
Loading

0 comments on commit 1092a0a

Please sign in to comment.