-
Notifications
You must be signed in to change notification settings - Fork 49
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
build: update infrastructure to use the uv tool (#1813)
Modernize development infrastructure to use newer python tools. The `uv` (https://docs.astral.sh/uv/) tool is used as project management tool, `nox` (https://nox.thea.codes/) replaces `tox` as the project test runner, custom scripts and `cog` snippets are replaced by a central `dev-tasks.py` tool with multiple commands. Additionally, documentation and CI workflows are updated accordingly and some tests which were not properly executed properly are fixed. Other related tasks to enhance the development infrastructure which could be done after this PR is merged are being tracked in issue #1829
- Loading branch information
Showing
51 changed files
with
4,181 additions
and
1,361 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,13 +13,17 @@ jobs: | |
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
|
|
||
| - name: Set up Python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: "3.10" | ||
| cache: 'pip' | ||
| cache-dependency-path: | | ||
| **/pyproject.toml | ||
| **/constraints.txt | ||
| **/requirements-dev.txt | ||
| - uses: pre-commit/[email protected] | ||
| python-version-file: ".python-version" | ||
|
|
||
| - name: Install uv | ||
| uses: astral-sh/setup-uv@v5 | ||
| with: | ||
| enable-cache: true | ||
| cache-dependency-glob: "uv.lock" | ||
|
|
||
| - name: "Run pre-commit" | ||
| uses: pre-commit/[email protected] | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,7 +5,8 @@ on: | |
| - cron: '0 4 * * *' | ||
| workflow_dispatch: | ||
|
|
||
| ## COMMENTED OUT: only for testing CI action changes | ||
| ## COMMENTED OUT: only for testing CI action changes. | ||
| ## It only works for PRs to `main` branch from branches in the upstream gt4py repo. | ||
| # pull_request: | ||
| # branches: | ||
| # - main | ||
|
|
@@ -15,106 +16,87 @@ jobs: | |
| daily-ci: | ||
| strategy: | ||
| matrix: | ||
| # dependencies-strategy -> The strategy that `uv lock` should use to select | ||
| # between the different compatible versions for a given package requirement | ||
| # [arg: --resolution, env: UV_RESOLUTION=] | ||
| dependencies-strategy: ["lowest-direct", "highest"] | ||
| gt4py-module: ["cartesian", "eve", "next", "storage"] | ||
| os: ["ubuntu-latest"] #, "macos-latest"] | ||
| python-version: ["3.10", "3.11"] | ||
| tox-module-factor: ["cartesian", "eve", "next", "storage"] | ||
| os: ["ubuntu-latest"] | ||
| requirements-file: ["requirements-dev.txt", "min-requirements-test.txt", "min-extra-requirements-test.txt"] | ||
| fail-fast: false | ||
|
|
||
| runs-on: ${{ matrix.os }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
|
|
||
| - name: Install C++ libraries | ||
| if: ${{ matrix.os == 'macos-latest' }} | ||
| shell: bash | ||
| run: | | ||
| brew install boost | ||
| run: brew install boost | ||
|
|
||
| - name: Install C++ libraries | ||
| if: ${{ matrix.os == 'ubuntu-latest' }} | ||
| shell: bash | ||
| run: | | ||
| sudo apt install libboost-dev | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| uses: actions/setup-python@v5 | ||
| run: sudo apt install libboost-dev | ||
|
|
||
| - name: Install uv and set the python version | ||
| uses: astral-sh/setup-uv@v5 | ||
| with: | ||
| enable-cache: true | ||
| cache-dependency-glob: "uv.lock" | ||
| python-version: ${{ matrix.python-version }} | ||
| cache: 'pip' | ||
| cache-dependency-path: | | ||
| **/pyproject.toml | ||
| **/constraints.txt | ||
| **/requirements-dev.txt | ||
| - name: Install tox | ||
| run: | | ||
| python -m pip install -c ./constraints.txt pip setuptools wheel tox | ||
| python -m pip list | ||
| - name: Update requirements | ||
| run: | | ||
| pyversion=${{ matrix.python-version }} | ||
| pyversion_no_dot=${pyversion//./} | ||
| tox run -e requirements-py${pyversion_no_dot} | ||
| # TODO(egparedes): add notification for dependencies updates | ||
| # - name: Check for updated requirements | ||
| # id: update-requirements | ||
| # continue-on-error: true | ||
| # if: ${{ matrix.python-version == '3.8' && matrix.tox-module-factor == 'cartesian' }} | ||
| # shell: bash | ||
| # run: | | ||
| # if diff -q constraints.txt CURRENT-constraints.txt; then | ||
| # echo "REQS_DIFF=''" >> $GITHUB_OUTPUT | ||
| # else | ||
| # diff --changed-group-format='%<' --unchanged-group-format='' constraints.txt CURRENT-constraints.txt | tr '\n' ' ' > constraints.txt.diff | ||
| # echo "REQS_DIFF='$(cat constraints.txt.diff)'" >> $GITHUB_OUTPUT | ||
| # fi | ||
| # echo "REQS_DIFF_TEST="FOOOOOOOO" >> $GITHUB_OUTPUT | ||
| # - name: Notify updated requirements (if any) | ||
| # if: ${{ steps.update-requirements.outputs.REQS_DIFF }} | ||
| # env: | ||
| # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | ||
| # uses: slackapi/[email protected] | ||
| # with: | ||
| # channel-id: ${{ vars.SLACK_BOT_CHANNEL }} | ||
| # payload: | | ||
| # { | ||
| # "text": "TEXT", | ||
| # "blocks": [ | ||
| # { | ||
| # "type": "section", | ||
| # "text": { | ||
| # "type": "plain_text", | ||
| # "text": "@channel: AA/${{ steps.update-requirements.outputs.REQS_DIFF }}/BB/ ${{ steps.update-requirements.outputs.REQS_DIFF_TEST }} /CC" | ||
| # } | ||
| # }, | ||
| # { | ||
| # "type": "section", | ||
| # "text": { | ||
| # "type": "mrkdwn", | ||
| # "text": "@channel: AA/${{ steps.update-requirements.outputs.REQS_DIFF }}/BB/ ${{ steps.update-requirements.outputs.REQS_DIFF_TEST }} /CC" | ||
| # } | ||
| # } | ||
| # ] | ||
| # } | ||
| - name: Run tests | ||
|
|
||
| - name: Run CPU tests for '${{ matrix.gt4py-module }}' with '${{ matrix.dependencies-strategy }}' resolution strategy | ||
| env: | ||
| NUM_PROCESSES: auto | ||
| ENV_REQUIREMENTS_FILE: ${{ matrix.requirements-file }} | ||
| run: | | ||
| tox run --skip-missing-interpreters -m test-${{ matrix.tox-module-factor }}-cpu | ||
| UV_RESOLUTION: ${{ matrix.dependencies-strategy }} | ||
| run: uv run nox -s 'test_${{ matrix.gt4py-module }}-${{ matrix.python-version }}' -t 'cpu' | ||
|
|
||
| - name: Notify slack | ||
| if: ${{ failure() }} | ||
| env: | ||
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | ||
| uses: slackapi/[email protected] | ||
| with: | ||
| channel-id: ${{ vars.SLACK_BOT_CHANNEL }} | ||
| channel-id: ${{ vars.SLACK_BOT_CHANNEL }} # Use SLACK_BOT_CHANNEL_TEST for testing | ||
| payload: | | ||
| { | ||
| "text": "Failed tests for ${{ github.workflow }} (dependencies-strategy=${{ matrix.dependencies-strategy }}, python=${{ matrix.python-version }}, component=${{ matrix.gt4py-module }}) [https://github.com/GridTools/gt4py/actions/runs/${{ github.run_id }}].", | ||
| "blocks": [ | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "Failed tests: <https://github.com/GridTools/gt4py/actions/runs/${{ github.run_id }}|${{ github.workflow }}: **${{ matrix.gt4py-module }} (CPU)** for **Python-${{ matrix.python-version }}** with **'${{ matrix.dependencies-strategy }}' uv resolution strategy**.>" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| weekly-reminder: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - id: get_day_of_the_week | ||
| name: Get day of the week | ||
| run: echo "day_of_week=$(date +'%u')" >> $GITHUB_OUTPUT | ||
|
|
||
| - name: Weekly notification | ||
| if: ${{ env.DAY_OF_WEEK == 1 }} | ||
| env: | ||
| DAY_OF_WEEK: ${{ steps.get_day_of_the_week.outputs.day_of_week }} | ||
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | ||
| uses: slackapi/[email protected] | ||
| with: | ||
| channel-id: ${{ vars.SLACK_BOT_CHANNEL }} # Use SLACK_BOT_CHANNEL_TEST for testing | ||
| payload: | | ||
| { | ||
| "text": "${{ github.workflow }}: `test-${{ matrix.tox-module-factor }}-cpu (python${{ matrix.python-version }})`>: *Failed tests!*", | ||
| "text": "Weekly reminder to check the latest runs of the GT4Py Daily CI workflow at the GitHub Actions dashboard [https://github.com/GridTools/gt4py/actions/workflows/daily-ci.yml].", | ||
| "blocks": [ | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "<https://github.com/GridTools/gt4py/actions/runs/${{ github.run_id }}|${{ github.workflow }}: `test-${{ matrix.tox-module-factor }}-cpu (python${{ matrix.python-version }})`>: *Failed tests!*" | ||
| "text": "Weekly reminder to check the latest runs of the <https://github.com/GridTools/gt4py/actions/workflows/daily-ci.yml|GT4Py Daily CI> workflow at the GitHub Actions dashboard." | ||
| } | ||
| } | ||
| ] | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.