Merge pull request #456 from hiddenSymmetries/ml/shifter_mpi4py_fix #308
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file is based on examples in | |
| # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions | |
| # Note that all the "sudo" commands here appear to cause a warning message | |
| # "sudo: setrlimit(RLIMIT_CORE): operation not permitted" | |
| # This appears to be a known harmless annoyance: | |
| # https://gitlab.alpinelinux.org/alpine/aports/-/issues/11122 | |
| name: Extensive CI | |
| on: | |
| push: | |
| branches: [master] | |
| tags: [v*] | |
| pull_request: | |
| branches: [master] | |
| workflow_dispatch: | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| env: | |
| OMPI_ALLOW_RUN_AS_ROOT: 1 | |
| OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test-type: [unit, integrated] | |
| packages: [all, vmec, spec, none] | |
| python-version: [3.9, "3.10", "3.11"] | |
| include: | |
| - python-version: 3.9 | |
| test-type: unit | |
| packages: none | |
| - python-version: 3.9 | |
| test-type: integrated | |
| packages: none | |
| steps: | |
| # First print out lots of information. We do this in separate | |
| # "name" blocks because otherwise the output gets mixed together | |
| # in the github actions log. | |
| - name: Print user and group id | |
| run: | | |
| set -ex | |
| id | |
| - name: PWD | |
| run: | | |
| set -ex | |
| pwd | |
| - name: ls -l | |
| run: | | |
| set -ex | |
| ls -l | |
| - name: apt-get stuff needed for libstell and vmec | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential gfortran openmpi-bin libopenmpi-dev libnetcdf-dev libnetcdff-dev liblapack-dev libscalapack-mpi-dev libhdf5-dev libhdf5-serial-dev git m4 libfftw3-dev libopenblas-dev libboost-all-dev | |
| - uses: actions/checkout@v4 | |
| # If we want submodules downloaded, uncomment the next 2 lines: | |
| #with: | |
| # submodules: true | |
| # We must run actions/checkout before downloading and building VMEC, since checkout deletes the contents of the directory. | |
| - name: Download the VMEC2000 standalone repository | |
| if: contains(matrix.packages, 'vmec') || contains(matrix.packages, 'all') | |
| run: git clone https://github.com/hiddensymmetries/VMEC2000.git | |
| - name: ls -l again | |
| run: | | |
| set -ex | |
| ls -l | |
| pwd | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: which python3 after python setup | |
| run: which python3 | |
| - name: which pip after python setup | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip --version | |
| - name: env after adding python | |
| run: env | |
| - name: Install python dependencies | |
| run: | | |
| sudo apt-get install graphviz graphviz-dev | |
| pip install wheel "numpy<2.0.0" scipy f90nml h5py scikit-build cmake qsc sympy pyevtk matplotlib ninja plotly networkx pygraphviz ground bentley_ottmann f90wrap | |
| - name: Install booz_xform | |
| if: contains(matrix.packages, 'vmec') || contains(matrix.packages, 'all') | |
| run: pip install -v git+https://github.com/hiddenSymmetries/booz_xform | |
| - name: Install virtual_casing | |
| run: pip install -v git+https://github.com/hiddenSymmetries/virtual-casing | |
| # See https://github.community/t/best-way-to-clone-a-private-repo-during-script-run-of-private-github-action/16116/7 | |
| # https://stackoverflow.com/questions/57612428/cloning-private-github-repository-within-organisation-in-actions | |
| # https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token | |
| - name: Check out SPEC | |
| if: contains(matrix.packages, 'spec') || contains(matrix.packages, 'all') | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: PrincetonUniversity/SPEC | |
| path: SPEC | |
| # For some reason, installing py_spec does not install the dependencies f90nml and h5py. Therefore I installed these manually above. | |
| - name: Install py_spec | |
| if: contains(matrix.packages, 'spec') || contains(matrix.packages, 'all') | |
| run: | | |
| pip install -r SPEC/Utilities/pythontools/requirements.txt | |
| pip install -e SPEC/Utilities/pythontools | |
| python -c "import py_spec; print('success')" | |
| - name: Build SPEC python wrapper. | |
| if: contains(matrix.packages, 'spec') || contains(matrix.packages, 'all') | |
| run: | | |
| cd SPEC | |
| pip install . | |
| - name: Try import spec | |
| if: contains(matrix.packages, 'spec') || contains(matrix.packages, 'all') | |
| run: python -c "import spec.spec_f90wrapped as spec; print(spec.constants.version)" | |
| - name: ls in /usr/lib/x86_64-linux-gnu | |
| run: ls -l /usr/lib/x86_64-linux-gnu | |
| - name: Add to LD_LIBRARY_PATH so scalapack etc can be found | |
| run: echo "LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV | |
| - name: env after adding to LD_LIBRARY_PATH | |
| run: env | |
| - name: ls in VMEC2000/python 1 | |
| if: contains(matrix.packages, 'vmec') || contains(matrix.packages, 'all') | |
| run: ls -l VMEC2000/python | |
| - name: Configure VMEC2000 module | |
| if: contains(matrix.packages, 'vmec') || contains(matrix.packages, 'all') | |
| run: | | |
| cd VMEC2000 | |
| cp cmake/machines/ubuntu.json cmake_config_file.json | |
| cat cmake_config_file.json | |
| pip install . | |
| - name: Try importing vmec module | |
| if: contains(matrix.packages, 'vmec') || contains(matrix.packages, 'all') | |
| run: python -c "print(dir()); import vmec; print(dir()); print(dir(vmec)); print('package:', vmec.__package__); print('spec:', vmec.__spec__); print('doc:', vmec.__doc__); print('file:', vmec.__file__); print('path:', vmec.__path__)" | |
| - name: Install simsopt package | |
| if: contains(matrix.packages, 'spec') || contains(matrix.packages, 'all') | |
| run: | | |
| pip install -v git+https://github.com/zhisong/pyoculus | |
| pip install -v . | |
| pip install mpi4py py_spec h5py | |
| - name: Install simsopt package | |
| if: contains(matrix.packages, 'none') | |
| run: pip install -v . | |
| - name: Install simsopt package | |
| if: contains(matrix.packages, 'vmec') | |
| run: | | |
| pip install -v . | |
| pip install mpi4py | |
| - name: Run serial examples as part of integrated tests | |
| if: contains(matrix.test-type, 'integrated') && (contains(matrix.packages, 'none') || contains(matrix.packages, 'all')) | |
| run: | | |
| cd examples | |
| ./run_serial_examples | |
| - name: Run simple parallel examples as part of integrated tests | |
| if: contains(matrix.test-type, 'integrated') && contains(matrix.packages, 'all') | |
| run: | | |
| cd examples | |
| ./run_parallel_examples | |
| - name: Run spec examples as part of integrated tests | |
| if: contains(matrix.test-type, 'integrated') && contains(matrix.packages, 'spec') | |
| run: | | |
| cd examples | |
| ./run_spec_examples | |
| - name: Run vmec examples as part of integrated tests | |
| if: contains(matrix.test-type, 'integrated') && contains(matrix.packages, 'vmec') | |
| run: | | |
| cd examples | |
| ./run_vmec_examples | |
| - name: Run vmec and spec examples as part of integrated tests | |
| if: contains(matrix.test-type, 'integrated') && contains(matrix.packages, 'all') | |
| run: | | |
| cd examples | |
| ./run_spec_vmec_examples | |
| - name: Install coverage dependencies | |
| if: contains(matrix.test-type, 'unit') | |
| run: pip install coverage | |
| - name: Run expensive tests on 1 MPI process using coverage | |
| if: contains(matrix.test-type, 'unit') && (contains(matrix.packages, 'none') || contains(matrix.packages, 'all')) | |
| run: | | |
| coverage run --source=simsopt -m unittest discover -t tests -v -s tests/geo | |
| coverage run --source=simsopt -m unittest discover -t tests -v -s tests/field | |
| - name: Run remaining tests on 1 MPI process using coverage | |
| if: contains(matrix.test-type, 'unit') | |
| run: | | |
| coverage run --source=simsopt -m unittest discover -t tests -v -s tests/core | |
| coverage run --source=simsopt -m unittest discover -t tests -v -s tests/mhd | |
| coverage run --source=simsopt -m unittest discover -t tests -v -s tests/objectives | |
| coverage run --source=simsopt -m unittest discover -t tests -v -s tests/solve | |
| coverage run --source=simsopt -m unittest discover -t tests -v -s tests/util | |
| - name: Run MPI tests using coverage | |
| if: contains(matrix.test-type, 'unit') && (contains(matrix.packages, 'all') || contains(matrix.packages, 'vmec') || contains(matrix.packages, 'spec')) | |
| run: | | |
| mpiexec -n 1 coverage run -m unittest discover -k "mpi" -s tests -v | |
| mpiexec -n 2 coverage run -m unittest discover -k "mpi" -s tests -v | |
| mpiexec -n 3 --oversubscribe coverage run -m unittest discover -k "mpi" -s tests -v | |
| - name: ls to see coverage files that were produced | |
| if: contains(matrix.test-type, 'unit') | |
| run: ls -al | |
| - name: Upload uncombined coverage to github | |
| if: contains(matrix.test-type, 'unit') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gh-actions-parallel-coverage-${{ matrix.packages }}-${{ matrix.python-version }} | |
| path: .coverage.* | |
| if-no-files-found: error | |
| include-hidden-files: true | |
| coverage: | |
| runs-on: ubuntu-latest | |
| needs: [ test ] | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 | |
| - name: Install coverage | |
| run: pip install coverage | |
| - name: Checkout simsopt | |
| uses: actions/checkout@v4 | |
| - name: Install simsopt before running coverage | |
| run: pip install . | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ./ | |
| merge-multiple: true | |
| - name: Display coverage files before combine | |
| run: ls -a | |
| - name: Combine coverage reports | |
| run: | | |
| coverage combine | |
| coverage report | |
| coverage xml | |
| - name: Upload coverage to github | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tox-gh-actions-coverage | |
| path: coverage.xml | |
| if-no-files-found: error | |
| - name: Upload coverage to Codecov | |
| # The next line prevents github from trying to upload to Codecov on forks of the repository, avoiding a permissions error | |
| if: github.repository_owner == 'hiddenSymmetries' | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: unittests | |
| env_vars: PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: true | |
| verbose: true |