diff --git a/.cruft.json b/.cruft.json index 866013e7..52912282 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/Ouranosinc/cookiecutter-pypackage", - "commit": "14556700478b0afdb158d61dd35db26a77c2b83d", + "commit": "36ea29394390254407194bd37315d9e3e9238585", "checkout": null, "context": { "cookiecutter": { @@ -11,7 +11,7 @@ "project_slug": "xscen", "project_short_description": "A climate change scenario-building analysis framework, built with xclim/xarray.", "pypi_username": "RondeauG", - "version": "0.10.2-dev.0", + "version": "0.10.2-dev.1", "use_pytest": "y", "use_black": "y", "use_conda": "y", @@ -23,7 +23,8 @@ "open_source_license": "Apache Software License 2.0", "generated_with_cruft": "y", "__gh_slug": "https://github.com/Ouranosinc/xscen", - "_template": "https://github.com/Ouranosinc/cookiecutter-pypackage" + "_template": "https://github.com/Ouranosinc/cookiecutter-pypackage", + "_commit": "36ea29394390254407194bd37315d9e3e9238585" } }, "directory": null diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 053707cf..dbeb4799 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,6 +17,9 @@ updates: schedule: interval: monthly groups: + ci: + patterns: + - "CI/*" python: patterns: - - "*" + - "pyproject.toml" diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 8307ec13..02d74817 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -47,7 +47,7 @@ jobs: actions: read steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block @@ -58,13 +58,14 @@ jobs: pypi.org:443 - name: Generate App Token id: token_generator - uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0 + uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1 with: app-id: ${{ secrets.OURANOS_HELPER_BOT_ID }} private-key: ${{ secrets.OURANOS_HELPER_BOT_KEY }} - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: + persist-credentials: false token: ${{ steps.token_generator.outputs.token }} - name: Set up Python3 uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 @@ -78,26 +79,20 @@ jobs: git_user_signingkey: true git_commit_gpgsign: true trust_level: 5 - - name: Current Version - run: | - CURRENT_VERSION="$(grep -E '__version__' src/xscen/__init__.py | cut -d ' ' -f3)" - echo "current_version=${CURRENT_VERSION}" - echo "CURRENT_VERSION=${CURRENT_VERSION}" >> $GITHUB_ENV - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt - name: Conditional Bump Version run: | - if [[ ${{ env.CURRENT_VERSION }} =~ -dev(\.\d+)? ]]; then + CURRENT_VERSION=$(bump-my-version show current_version) + if [[ ${CURRENT_VERSION} =~ -dev(\.\d+)? ]]; then echo "Development version (ends in 'dev(\.\d+)?'), bumping 'build' version" bump-my-version bump build else echo "Version is stable, bumping 'patch' version" bump-my-version bump patch fi - NEW_VERSION="$(grep -E '__version__' src/xscen/__init__.py | cut -d ' ' -f3)" - echo "new_version=${NEW_VERSION}" - echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV + echo "new_version=$(bump-my-version show current_version)" - name: Push Changes uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df # v0.8.0 with: diff --git a/.github/workflows/cache-cleaner.yml b/.github/workflows/cache-cleaner.yml index 15ffca69..62f5ab87 100644 --- a/.github/workflows/cache-cleaner.yml +++ b/.github/workflows/cache-cleaner.yml @@ -5,7 +5,7 @@ on: types: - closed -permissions: # added using https://github.com/step-security/secure-repo +permissions: contents: read jobs: @@ -16,7 +16,7 @@ jobs: actions: write steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block @@ -27,6 +27,8 @@ jobs: - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Cleanup run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index dc2fd928..00000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: "CodeQL Scan" - -on: - push: - branches: - - main - pull_request: - schedule: - - cron: '30 23 * * 5' - -permissions: - contents: read - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - strategy: - fail-fast: false - matrix: - language: - - 'python' - steps: - - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 - with: - disable-sudo: true - egress-policy: block - allowed-endpoints: > - api.github.com:443 - files.pythonhosted.org:443 - github.com:443 - objects.githubusercontent.com:443 - pypi.org:443 - uploads.github.com:443 - - name: Checkout repository - uses: actions/checkout@v4.2.2 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@codeql-bundle-20230524 - with: - languages: ${{ matrix.language }} - - name: Autobuild - uses: github/codeql-action/autobuild@codeql-bundle-20230524 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@codeql-bundle-20230524 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..ec93722c --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,99 @@ +# For most projects, this workflow file will not need changing; you simply need to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL Advanced" + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '36 9 * * 1' + +permissions: + contents: read + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Harden Runner + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 + with: + disable-sudo: true + egress-policy: audit + + - name: Checkout Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: + # https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index a73a7e80..7756eb3b 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block @@ -28,6 +28,8 @@ jobs: - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Dependency Review - uses: actions/dependency-review-action@4081bf99e2866ebe428fc0477b69eb4fcda7220a # v4.4.0 + uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0 diff --git a/.github/workflows/first-pull-request.yml b/.github/workflows/first-pull-request.yml index 4bd31a3f..64aa58be 100644 --- a/.github/workflows/first-pull-request.yml +++ b/.github/workflows/first-pull-request.yml @@ -16,7 +16,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml index d7b02305..388d9d13 100644 --- a/.github/workflows/label.yml +++ b/.github/workflows/label.yml @@ -22,7 +22,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7d883d80..4450e8ce 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,7 +30,7 @@ jobs: - "3.x" steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block @@ -40,6 +40,8 @@ jobs: pypi.org:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Setup Python${{ matrix.python-version }} uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: @@ -64,18 +66,21 @@ jobs: matrix: os: [ 'ubuntu-latest' ] python-version: [ "3.10", "3.11", "3.12" ] # "3.13" + xclim-min-version: [ "0.53.2" ] defaults: run: shell: bash -l {0} steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: egress-policy: audit - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Setup Conda (Micromamba) with Python ${{ matrix.python-version }} - uses: mamba-org/setup-micromamba@617811f69075e3fd3ae68ca64220ad065877f246 # v2.0.0 + uses: mamba-org/setup-micromamba@0dea6379afdaffa5d528b3d1dabc45da37f443fc # v2.0.4 with: cache-downloads: true environment-name: xscen-pypi @@ -84,10 +89,8 @@ jobs: python=${{ matrix.python-version }} tox>=4.17.1 tox-gh>=1.3.2 - # FIXME: https://github.com/mamba-org/setup-micromamba/issues/225 - micromamba-version: "1.5.10-0" # pinned to avoid the breaking changes with mamba and micromamba (2.0.0). - name: Environment Caching - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: .tox key: ${{ matrix.os }}-Python${{ matrix.python-version }}-${{ hashFiles('pyproject.toml', 'tox.ini') }} @@ -96,6 +99,7 @@ jobs: python -m tox env: ESMF_VERSION: ${{ env.esmf-version }} + XCLIM_VERSION: ${{ matrix.xclim-min-version }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }} COVERALLS_PARALLEL: true @@ -137,24 +141,21 @@ jobs: shell: bash -l {0} steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: egress-policy: audit - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Setup Conda (Micromamba) with Python ${{ matrix.python-version }} - uses: mamba-org/setup-micromamba@617811f69075e3fd3ae68ca64220ad065877f246 # v2.0.0 + uses: mamba-org/setup-micromamba@0dea6379afdaffa5d528b3d1dabc45da37f443fc # v2.0.4 with: cache-downloads: true cache-environment: false # FIXME: No environment caching until issues with micromamba 2.0.0 are resolved. environment-file: environment-dev.yml create-args: >- python=${{ matrix.python-version }} - # FIXME: https://github.com/mamba-org/setup-micromamba/issues/225 - micromamba-version: "1.5.10-0" # pinned to avoid the breaking changes with mamba and micromamba (2.0.0). - - name: Conda and Mamba versions - run: | - echo "micromamba $(micromamba --version)" - name: Compile catalogs and install xscen run: | make translate @@ -182,7 +183,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: audit diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 6a2790b8..d8916873 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -18,19 +18,21 @@ jobs: id-token: write steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: -# disable-sudo: true - egress-policy: audit -# allowed-endpoints: > -# files.pythonhosted.org:443 -# fulcio.sigstore.dev:443 -# github.com:443 -# pypi.org:443 -# tuf-repo-cdn.sigstore.dev:443 -# upload.pypi.org:443 + disable-sudo: true + egress-policy: block + allowed-endpoints: > + files.pythonhosted.org:443 + fulcio.sigstore.dev:443 + github.com:443 + pypi.org:443 + ruf-repo-cdn.sigstore.dev:443 + upload.pypi.org:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Set up Python3 uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: @@ -42,4 +44,4 @@ jobs: run: | python -m build --sdist --wheel - name: Publish distribution đŸ“Ļ to PyPI - uses: pypa/gh-action-pypi-publish@fb13cb306901256ace3dab689990e13a5550ffaa # v1.11.0 + uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 0ecb2632..3465c6e0 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -16,7 +16,9 @@ on: - main # Declare default permissions as read only. -permissions: read-all +# Read-all permission is not technically needed for this workflow. +permissions: + contents: read jobs: analysis: @@ -29,7 +31,7 @@ jobs: id-token: write steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block @@ -72,7 +74,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: Upload artifact - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: SARIF file path: results.sarif @@ -80,6 +82,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: Upload to code-scanning - uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # 3.26.6 + uses: github/codeql-action/upload-sarif@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # 3.28.1 with: sarif_file: results.sarif diff --git a/.github/workflows/tag-testpypi.yml b/.github/workflows/tag-testpypi.yml index f0bba0f6..950388d3 100644 --- a/.github/workflows/tag-testpypi.yml +++ b/.github/workflows/tag-testpypi.yml @@ -17,7 +17,7 @@ jobs: contents: write steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block @@ -26,8 +26,10 @@ jobs: github.com:443 - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Create Release - uses: softprops/action-gh-release@e7a8f85e1c67a31e6ed99a94b41bd0b71bbee6b8 # 2.0.9 + uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # 2.2.1 env: # This token is provided by Actions, you do not need to create your own token GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -46,19 +48,21 @@ jobs: id-token: write steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: -# disable-sudo: true + disable-sudo: true egress-policy: audit -# allowed-endpoints: > -# files.pythonhosted.org:443 -# fulcio.sigstore.dev:443 -# github.com:443 -# pypi.org:443 -# test.pypi.org:443 -# tuf-repo-cdn.sigstore.dev:443 + allowed-endpoints: > + files.pythonhosted.org:443 + fulcio.sigstore.dev:443 + github.com:443 + pypi.org:443 + ruf-repo-cdn.sigstore.dev:443 + test.pypi.org:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Set up Python3 uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: @@ -70,7 +74,7 @@ jobs: run: | python -m build --sdist --wheel - name: Publish distribution đŸ“Ļ to Test PyPI - uses: pypa/gh-action-pypi-publish@fb13cb306901256ace3dab689990e13a5550ffaa # v1.11.0 + uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 with: repository-url: https://test.pypi.org/legacy/ skip-existing: true diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index a596b09f..1712ec8a 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -40,15 +40,16 @@ jobs: shell: bash -l {0} steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: egress-policy: audit - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 # Fetch all history for all branches and tags. + persist-credentials: false - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} - uses: mamba-org/setup-micromamba@617811f69075e3fd3ae68ca64220ad065877f246 # v2.0.0 + uses: mamba-org/setup-micromamba@0dea6379afdaffa5d528b3d1dabc45da37f443fc # v2.0.4 with: cache-downloads: true cache-environment: true diff --git a/.github/workflows/workflow-warning.yml b/.github/workflows/workflow-warning.yml index 0a1e658c..6933e1b6 100644 --- a/.github/workflows/workflow-warning.yml +++ b/.github/workflows/workflow-warning.yml @@ -25,7 +25,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: disable-sudo: true egress-policy: block diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 12d5339f..7224ed20 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: https://github.com/asottile/pyupgrade - rev: v3.19.0 + rev: v3.19.1 hooks: - id: pyupgrade args: [ '--py310-plus' ] @@ -31,7 +31,7 @@ repos: - id: python-use-type-annotations - id: rst-inline-touching-normal - repo: https://github.com/pappasam/toml-sort - rev: v0.23.1 + rev: v0.24.2 hooks: - id: toml-sort-fix - repo: https://github.com/psf/black-pre-commit-mirror @@ -45,7 +45,7 @@ repos: - id: isort exclude: ^docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.2 + rev: v0.8.6 hooks: - id: ruff args: [ '--fix', '--show-fixes' ] @@ -56,12 +56,21 @@ repos: - id: flake8 additional_dependencies: [ 'flake8-rst-docstrings' ] args: [ '--config=.flake8' ] + - repo: https://github.com/jendrikseipp/vulture + rev: v2.13 + hooks: + - id: vulture +# - repo: https://github.com/pre-commit/mirrors-mypy +# rev: v1.14.1 +# hooks: +# - id: mypy - repo: https://github.com/keewis/blackdoc rev: v0.3.9 hooks: - id: blackdoc - additional_dependencies: [ 'black==24.4.2' ] + additional_dependencies: [ 'black==24.10.0' ] exclude: config.py + - id: blackdoc-autoupdate-black - repo: https://github.com/adrienverge/yamllint.git rev: v1.35.1 hooks: @@ -73,18 +82,18 @@ repos: # - id: numpydoc-validation # exclude: ^docs/|^tests/ - repo: https://github.com/nbQA-dev/nbQA - rev: 1.8.7 + rev: 1.9.1 hooks: - id: nbqa-pyupgrade args: [ '--py310-plus' ] additional_dependencies: [ 'pyupgrade==3.17.0' ] - id: nbqa-black args: [ '--target-version=py310' ] - additional_dependencies: [ 'black==24.8.0' ] + additional_dependencies: [ 'black==24.10.0' ] - id: nbqa-isort additional_dependencies: [ 'isort==5.13.2' ] - repo: https://github.com/kynan/nbstripout - rev: 0.8.0 + rev: 0.8.1 hooks: - id: nbstripout files: ".ipynb" @@ -96,10 +105,19 @@ repos: exclude: .cruft.json|docs/notebooks args: [ '--baseline=.secrets.baseline' ] - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.30.0 hooks: - id: check-github-workflows - id: check-readthedocs + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v0.8.0 + hooks: + - id: zizmor + args: [ '--config=.zizmor.yml' ] + - repo: https://github.com/gitleaks/gitleaks + rev: v8.21.2 + hooks: + - id: gitleaks - repo: meta hooks: - id: check-hooks-apply diff --git a/.zizmor.yml b/.zizmor.yml new file mode 100644 index 00000000..6ac32154 --- /dev/null +++ b/.zizmor.yml @@ -0,0 +1,6 @@ +rules: + dangerous-triggers: + ignore: + - label.yml:9 + - first-pull-request.yml:3 + - workflow-warning.yml:3 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c20ce8d0..855b6b0c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,11 +4,11 @@ Changelog v0.11.0 (unreleased) -------------------- -Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`). +Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`), Juliette Lavoie (:user:`juliettelavoie`), Trevor James Smith (:user:`Zeitsperre`). New features and enhancements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* N/A +* Improve ``xs.ensembles.build_partition_data``. (:pull:`504`). Breaking changes ^^^^^^^^^^^^^^^^ @@ -29,7 +29,13 @@ Internal changes * The estimation method in ``xs.io.estimate_chunks`` has been improved. (:pull:`492`). * A new parameter `incomplete` has been added to ``xs.io.clean_incomplete`` to remove incomplete variables. (:pull:`492`). * Continued work on adding tests. (:pull:`492`). - +* Modified a CI build to test against the oldest supported version of `xclim`. (:pull:`505`). +* Updated the cookiecutter template version: (:pull:`507`) + * Added `vulture` to pre-commit hooks (finding dead code blocks). + * Added `zizmor` to the pre-commit hooks (security analysis for CI workflows). + * Secured token usages on all workflows (using `zizmor`). + * Simplified logic in ``bump-version.yml``. + * Synchronized a few dependencies. v0.10.1 (2024-11-04) -------------------- diff --git a/MANIFEST.in b/MANIFEST.in index 070cdb41..842dbdf2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -34,6 +34,7 @@ exclude .pre-commit-config.yaml exclude .readthedocs.yml exclude .secrets.baseline exclude .yamllint.yaml +exclude .zizmor.yml exclude environment.yml exclude environment-dev.yml exclude tox.ini diff --git a/docs/conf.py b/docs/conf.py index 20e10c3b..a419d3c8 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,6 +23,7 @@ import warnings from datetime import datetime from pathlib import Path +from typing import Any sys.path.insert(0, os.path.abspath("..")) if os.environ.get("READTHEDOCS") and "ESMFMKFILE" not in os.environ: @@ -222,7 +223,7 @@ # -- Options for LaTeX output ------------------------------------------ -latex_elements = { +latex_elements: dict[str, Any] = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', diff --git a/docs/notebooks/4_ensembles.ipynb b/docs/notebooks/4_ensembles.ipynb index fa76d1ac..0fbf8855 100644 --- a/docs/notebooks/4_ensembles.ipynb +++ b/docs/notebooks/4_ensembles.ipynb @@ -169,19 +169,28 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "is_executing": true + }, "outputs": [], "source": [ "# Get catalog\n", "from pathlib import Path\n", "\n", + "import xclim as xc\n", + "\n", "output_folder = Path().absolute() / \"_data\"\n", "cat = xs.DataCatalog(str(output_folder / \"tutorial-catalog.json\"))\n", "\n", "# create a dictionnary of datasets wanted for the partition\n", "input_dict = cat.search(variable=\"tas\", member=\"r1i1p1f1\").to_dataset_dict(\n", " xarray_open_kwargs={\"engine\": \"h5netcdf\"}\n", - ")" + ")\n", + "datasets = {}\n", + "for k, v in input_dict.items():\n", + " ds = xc.atmos.tg_mean(v.tas).to_dataset()\n", + " ds.attrs = v.attrs\n", + " datasets[k] = ds" ] }, { @@ -204,9 +213,8 @@ "import xclim as xc\n", "\n", "ds = xs.ensembles.build_partition_data(\n", - " input_dict,\n", + " datasets,\n", " subset_kw=dict(name=\"mtl\", method=\"gridpoint\", lat=[45.5], lon=[-73.6]),\n", - " indicators_kw={\"indicators\": [xc.atmos.tg_mean]},\n", ")\n", "ds" ] diff --git a/environment-dev.yml b/environment-dev.yml index 56345653..c27dbe17 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -30,20 +30,22 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray >=2023.11.0, !=2024.6.0 - - xclim >=0.53.2, <0.54 + - xarray >=2023.11.0, !=2024.6.0, <2024.10.0 # FIXME: 2024.10.0 breaks rechunker with zarr, https://github.com/pangeo-data/rechunker/issues/154 + - xclim >=0.53.2, <0.55 - xesmf >=0.7, <0.8.8 # FIXME: 0.8.8 currently creates segfaults on ReadTheDocs. - - zarr >=2.13 + - zarr >=2.13, <3.0 # FIXME: xarray is compatible with zarr 3.0 from 2025.01.1, but we pin xarray below that version # Opt - nc-time-axis >=1.3.1 - pyarrow >=10.0.1 # Dev - babel - - black ==24.8.0 + - pip >=24.3.1 + - black ==24.10.0 - blackdoc ==0.3.9 - - bump-my-version >=0.26.8 - - coverage>=7.5.0 - - coveralls>=4.0.1 + - bump-my-version >=0.28.0 + - click >=8.1.7 + - coverage >=7.5.0 + - coveralls >=4.0.1 - flake8 >=7.1.0 - flake8-rst-docstrings>=0.3.0 - ipykernel @@ -56,10 +58,11 @@ dependencies: - pandoc - pooch - pre-commit >=3.5.0 + - pygments <2.19 #FIXME: temporary fix, https://github.com/felix-hilden/sphinx-codeautolink/issues/153 - pytest >=8.3.2 - pytest-cov >=5.0.0 - pytest-xdist >=3.2.0 - - ruff >=0.5.7 + - ruff >=0.8.2 - setuptools >=65.0.0 - setuptools-scm >=8.0.0 - sphinx >=7.0.0 @@ -72,7 +75,7 @@ dependencies: - watchdog >=4.0.0 - xdoctest # Testing - - tox >=4.17.1 + - tox >=4.23.2 - tox-gh >=1.3.2 # packaging - conda-build diff --git a/environment.yml b/environment.yml index 126cb271..e545bfe8 100644 --- a/environment.yml +++ b/environment.yml @@ -30,10 +30,10 @@ dependencies: - shapely >=2.0 - sparse - toolz - - xarray >=2023.11.0, !=2024.6.0 - - xclim >=0.53.2, <0.54 + - xarray >=2023.11.0, !=2024.6.0, <2024.10.0 # FIXME: 2024.10.0 breaks rechunker with zarr + - xclim >=0.53.2, <0.55 - xesmf >=0.7, <0.8.8 # FIXME: 0.8.8 currently creates segfaults on ReadTheDocs. - - zarr >=2.13 + - zarr >=2.13, <3.0 # FIXME: xarray is compatible with zarr 3.0 from 2025.01.1, but we pin xarray below that version # To install from source - setuptools >=65.0.0 - setuptools-scm >=8.0.0 diff --git a/pyproject.toml b/pyproject.toml index a46c4883..1aa9b7ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,19 +65,18 @@ dependencies = [ "shapely >=2.0", "sparse", "toolz", - "xarray >=2023.11.0, !=2024.6.0", - "xclim >=0.53.2, <0.54", - "zarr >=2.13" + "xarray >=2023.11.0, !=2024.6.0, <2024.10.0", # FIXME: 2024.10.0 breaks rechunker with zarr + "xclim >=0.53.2, <0.55", + "zarr >=2.13, <3.0" # FIXME: xarray is compatible with zarr 3.0 from 2025.01.1, but we pin xarray below that version" ] [project.optional-dependencies] dev = [ # Dev tools and testing - "pip >=24.2.0", "babel", - "black[jupyter] ==24.10.0", + "black ==24.10.0", "blackdoc ==0.3.9", - "bump-my-version >=0.26.0", + "bump-my-version >=0.28.0", "coverage >=7.5.0", "coveralls >=4.0.1", "flake8 >=7.1.1", @@ -86,12 +85,12 @@ dev = [ "mypy", "numpydoc >=1.8.0", "pooch >=1.8.0", - "pre-commit >=3.3.2", + "pre-commit >=3.5.0", "pytest-cov >=5.0.0", "pytest >=8.3.2", "pytest-xdist[psutil] >=3.2.0", - "ruff >=0.5.7", - "tox >=4.18.0", + "ruff >=0.8.2", + "tox >=4.23.2", "watchdog >=4.0.0", "xdoctest" ] @@ -109,7 +108,8 @@ docs = [ "sphinx-intl", "sphinx-mdinclude", "sphinx-rtd-theme >=1.0", - "sphinxcontrib-napoleon" + "sphinxcontrib-napoleon", + "pygments <2.19" # FIXME: temporary fix, https://github.com/felix-hilden/sphinx-codeautolink/issues/153 ] extra = [ "xesmf>=0.7, <0.8.8" # FIXME: 0.8.8 currently creates segfaults on ReadTheDocs. @@ -134,7 +134,7 @@ target-version = [ ] [tool.bumpversion] -current_version = "0.10.2-dev.0" +current_version = "0.10.2-dev.1" commit = true commit_args = "--no-verify" tag = false @@ -320,6 +320,14 @@ version = {attr = "xscen.__version__"} where = ["src"] include = ["xscen"] +[tool.vulture] +exclude = [] +ignore_decorators = ["@pytest.fixture"] +ignore_names = [] +min_confidence = 90 +paths = ["src/xscen", "tests"] +sort_by_size = true + # [tool.setuptools.packages.find] # include = [ # ".zenodo.json", diff --git a/src/xscen/__init__.py b/src/xscen/__init__.py index d07afc36..f6e94b57 100644 --- a/src/xscen/__init__.py +++ b/src/xscen/__init__.py @@ -71,11 +71,17 @@ __author__ = """Gabriel Rondeau-Genesse""" __email__ = "rondeau-genesse.gabriel@ouranos.ca" -__version__ = "0.10.2-dev.0" +__version__ = "0.10.2-dev.1" +# FIXME: file and line are unused def warning_on_one_line( - message: str, category: Warning, filename: str, lineno: int, file=None, line=None + message: str, + category: Warning, + filename: str, + lineno: int, + file=None, # noqa: F841 + line=None, # noqa: F841 ): """ Monkeypatch Reformat warning so that `warnings.warn` doesn't mention itself. diff --git a/src/xscen/data/fr/LC_MESSAGES/xscen.mo b/src/xscen/data/fr/LC_MESSAGES/xscen.mo index 51b5812a..a8cfcf25 100644 Binary files a/src/xscen/data/fr/LC_MESSAGES/xscen.mo and b/src/xscen/data/fr/LC_MESSAGES/xscen.mo differ diff --git a/src/xscen/ensembles.py b/src/xscen/ensembles.py index cc55fb99..15df069c 100644 --- a/src/xscen/ensembles.py +++ b/src/xscen/ensembles.py @@ -12,10 +12,12 @@ import xarray as xr from xclim import ensembles +from .catalog import DataCatalog +from .catutils import generate_id from .config import parse_config from .indicators import compute_indicators from .regrid import regrid_dataset -from .spatial import subset +from .spatial import get_grid_mapping, subset from .utils import clean_up, get_cat_attrs logger = logging.getLogger(__name__) @@ -629,13 +631,167 @@ def generate_weights( # noqa: C901 return weights +def _partition_from_list(datasets, partition_dim, subset_kw, regrid_kw): + list_ds = [] + # only keep attrs common to all datasets + common_attrs = False + for ds in datasets: + if subset_kw: + ds = subset(ds, **subset_kw) + gridmap = get_grid_mapping(ds) + ds = ds.drop_vars( + [ + ds.cf["longitude"].name, + ds.cf["latitude"].name, + ds.cf.axes["X"][0], + ds.cf.axes["Y"][0], + gridmap, + ], + errors="ignore", + ) + + if regrid_kw: + ds = regrid_dataset(ds, **regrid_kw) + + for dim in partition_dim: + if f"cat:{dim}" in ds.attrs: + ds = ds.expand_dims(**{dim: [ds.attrs[f"cat:{dim}"]]}) + + if "bias_adjust_project" in ds.dims: + ds = ds.assign_coords( + adjustment=( + "bias_adjust_project", + [ds.attrs.get("cat:adjustment", np.nan)], + ) + ) + ds = ds.assign_coords( + reference=( + "bias_adjust_project", + [ds.attrs.get("cat:reference", np.nan)], + ) + ) + + if "realization" in partition_dim: + new_source = f"{ds.attrs['cat:institution']}_{ds.attrs['cat:source']}_{ds.attrs['cat:member']}" + ds = ds.expand_dims(realization=[new_source]) + + a = ds.attrs + a.pop("intake_esm_vars", None) # remove list for intersection to work + common_attrs = dict(common_attrs.items() & a.items()) if common_attrs else a + list_ds.append(ds) + ens = xr.merge(list_ds) + ens.attrs = common_attrs + return ens + + +def _partition_from_catalog( + datasets, partition_dim, subset_kw, regrid_kw, to_dataset_kw +): + + if ("adjustment" in partition_dim or "reference" in partition_dim) and ( + "bias_adjust_project" in partition_dim + ): + raise ValueError( + "The partition_dim can have either adjustment and reference or bias_adjust_project, not both." + ) + + if ("realization" in partition_dim) and ("source" in partition_dim): + raise ValueError( + "The partition_dim can have either realization or source, not both." + ) + + # special case to handle source (create one dimension with institution_source_member) + ensemble_on_list = None + if "realization" in partition_dim: + partition_dim.remove("realization") + ensemble_on_list = ["institution", "source", "member"] + + subcat = datasets + + # get attrs that are common to all datasets + common_attrs = {} + for col, series in subcat.df.items(): + if (series[0] == series).all(): + common_attrs[f"cat:{col}"] = series[0] + + col_id = [ + ( + "adjustment" if "adjustment" in partition_dim else None + ), # instead of bias_adjust_project, need to use adjustment, not method bc .sel + ( + "reference" if "reference" in partition_dim else None + ), # instead of bias_adjust_project + "bias_adjust_project" if "bias_adjust_project" in partition_dim else None, + "mip_era", + "activity", + "driving_model", + "institution" if "realization" in partition_dim else None, + "source", + "experiment", + "member" if "realization" in partition_dim else None, + "domain", + ] + + subcat.df["id"] = generate_id(subcat.df, col_id) + + # create a dataset for each bias_adjust_project, modify grid and concat them + # choose dim that exists in partition_dim and first in the order of preference + order_of_preference = ["reference", "bias_adjust_project", "source"] + dim_with_different_grid = list(set(partition_dim) & set(order_of_preference))[0] + + list_ds = [] + for d in subcat.df[dim_with_different_grid].unique(): + ds = subcat.search(**{dim_with_different_grid: d}).to_dataset( + concat_on=partition_dim, + create_ensemble_on=ensemble_on_list, + **to_dataset_kw, + ) + + if subset_kw: + ds = subset(ds, **subset_kw) + gridmap = get_grid_mapping(ds) + ds = ds.drop_vars( + [ + ds.cf["longitude"].name, + ds.cf["latitude"].name, + ds.cf.axes["X"][0], + ds.cf.axes["Y"][0], + gridmap, + ], + errors="ignore", + ) + + if regrid_kw: + ds = regrid_dataset(ds, **regrid_kw) + + # add coords adjustment and reference + if "bias_adjust_project" in ds.dims: + ds = ds.assign_coords( + adjustment=( + "bias_adjust_project", + [ds.attrs.get("cat:adjustment", np.nan)], + ) + ) # need to use adjustment, not method bc .sel + ds = ds.assign_coords( + reference=( + "bias_adjust_project", + [ds.attrs.get("cat:reference", np.nan)], + ) + ) + list_ds.append(ds) + ens = xr.concat(list_ds, dim=dim_with_different_grid) + ens.attrs = common_attrs + return ens + + def build_partition_data( datasets: dict | list[xr.Dataset], - partition_dim: list[str] = ["source", "experiment", "bias_adjust_project"], + partition_dim: list[str] = ["realization", "experiment", "bias_adjust_project"], subset_kw: dict | None = None, regrid_kw: dict | None = None, - indicators_kw: dict | None = None, rename_dict: dict | None = None, + to_dataset_kw: dict | None = None, + to_level: str = "partition-ensemble", ): """ Get the input for the xclim partition functions. @@ -644,16 +800,20 @@ def build_partition_data( `partition_dim` dimensions (and time) to pass to one of the xclim partition functions (https://xclim.readthedocs.io/en/stable/api.html#uncertainty-partitioning). If the inputs have different grids, - they have to be subsetted and regridded to a common grid/point. - Indicators can also be computed before combining the datasets. + they have to be subsetted and/or regridded to a common grid/point. Parameters ---------- - datasets : dict - List or dictionnary of Dataset objects that will be included in the ensemble. + datasets : list[xr.Dataset], dict[str, xr.Dataset], DataCatalog + Either a list/dictionary of Datasets or a DataCatalog that will be included in the ensemble. The datasets should include the necessary ("cat:") attributes to understand their metadata. - Tip: With a project catalog, you can do: `datasets = pcat.search(**search_dict).to_dataset_dict()`. - partition_dim : list[str] + Tip: A dictionary can be created with `datasets = pcat.search(**search_dict).to_dataset_dict()`. + + The use of a DataCatalog is recommended for large ensembles. + In that case, the ensembles will be loaded separately for each `bias_adjust_project`, + the subsetting or regridding can be applied before combining the datasets through concatenation. + If `bias_adjust_project` is not in `partition_dim`, `source` will be used instead. + partition_dim: list[str] Components of the partition. They will become the dimension of the output. The default is ['source', 'experiment', 'bias_adjust_project']. For source, the dimension will actually be institution_source_member. @@ -661,12 +821,15 @@ def build_partition_data( Arguments to pass to `xs.spatial.subset()`. regrid_kw : dict, optional Arguments to pass to `xs.regrid_dataset()`. - indicators_kw : dict, optional - Arguments to pass to `xs.indicators.compute_indicators()`. - All indicators have to be for the same frequency, in order to be put on a single time axis. + Note that regriding is computationally expensive. For large datasets, + it might be worth it to do the regridding first, outside of this function. rename_dict : dict, optional Dictionary to rename the dimensions from xscen names to xclim names. - If None, the default is {'source': 'model', 'bias_adjust_project': 'downscaling', 'experiment': 'scenario'}. + The default is {'source': 'model', 'bias_adjust_project': 'downscaling', 'experiment': 'scenario'}. + to_dataset_kw : dict, optional + Arguments to pass to `xscen.DataCatalog.to_dataset()` if datasets is a DataCatalog. + to_level: str + The processing level of the output dataset. Default is 'partition-ensemble'. Returns ------- @@ -682,41 +845,32 @@ def build_partition_data( # initialize dict subset_kw = subset_kw or {} regrid_kw = regrid_kw or {} + to_dataset_kw = to_dataset_kw or {} - list_ds = [] - for ds in datasets: - if subset_kw: - ds = subset(ds, **subset_kw) - - if regrid_kw: - ds = regrid_dataset(ds, **regrid_kw) - - if indicators_kw: - dict_ind = compute_indicators(ds, **indicators_kw) - if len(dict_ind) > 1: - raise ValueError( - f"The indicators computation should return only indicators of the same frequency.Returned frequencies: {dict_ind.keys()}" - ) - else: - ds = list(dict_ind.values())[0] + if isinstance(datasets, list): + ens = _partition_from_list(datasets, partition_dim, subset_kw, regrid_kw) - for dim in partition_dim: - if f"cat:{dim}" in ds.attrs: - ds = ds.expand_dims(**{dim: [ds.attrs[f"cat:{dim}"]]}) + elif isinstance(datasets, DataCatalog): + ens = _partition_from_catalog( + datasets, partition_dim, subset_kw, regrid_kw, to_dataset_kw + ) - if "source" in partition_dim: - new_source = f"{ds.attrs['cat:institution']}_{ds.attrs['cat:source']}_{ds.attrs['cat:member']}" - ds = ds.assign_coords(source=[new_source]) - list_ds.append(ds) - ens = xr.merge(list_ds) + else: + raise ValueError( + "'datasets' should be a list/dictionary of xarray datasets or a xscen.DataCatalog" + ) rename_dict = rename_dict or {} + rename_dict.setdefault("realization", "model") rename_dict.setdefault("source", "model") rename_dict.setdefault("experiment", "scenario") rename_dict.setdefault("bias_adjust_project", "downscaling") rename_dict = {k: v for k, v in rename_dict.items() if k in ens.dims} ens = ens.rename(rename_dict) + ens.attrs["cat:processing_level"] = to_level + ens.attrs["cat:id"] = generate_id(ens)[0] + return ens diff --git a/src/xscen/io.py b/src/xscen/io.py index 5186d7d4..c7935f3e 100644 --- a/src/xscen/io.py +++ b/src/xscen/io.py @@ -1065,7 +1065,6 @@ def rechunk( raise ValueError( "No chunks given. Need to give at `chunks_over_var` or `chunks_over_dim`." ) - plan = _rechunk(ds, chunks, worker_mem, str(path_out), temp_store=str(temp_store)) plan.execute() diff --git a/src/xscen/scripting.py b/src/xscen/scripting.py index 68e3677b..bc9fe808 100644 --- a/src/xscen/scripting.py +++ b/src/xscen/scripting.py @@ -297,7 +297,8 @@ def timeout(seconds: int, task: str = ""): yield else: - def _timeout_handler(signum, frame): + # FIXME: These variables are not used + def _timeout_handler(signum, frame): # noqa: F841 raise TimeoutException(seconds, task) old_handler = signal.signal(signal.SIGALRM, _timeout_handler) diff --git a/tests/test_biasadjust.py b/tests/test_biasadjust.py index b80ccdb2..8b86f003 100644 --- a/tests/test_biasadjust.py +++ b/tests/test_biasadjust.py @@ -46,12 +46,13 @@ def test_basic_train(self, var, period): np.testing.assert_array_equal(out["scaling"], result) def test_preprocess(self): - - dref360 = self.dref.convert_calendar("360_day", align_on="year") + # FIXME: put back the test when xclim 0.55 is released, https://github.com/Ouranosinc/xclim/pull/2038/files + # dhist360 = self.dhist.convert_calendar("360_day", align_on="year") + dhist360 = self.dhist.convert_calendar("noleap", align_on="year") out = xs.train( - dref360, - self.dhist, + self.dref, + dhist360, var="tas", period=["2001", "2002"], adapt_freq={"thresh": "2 K"}, diff --git a/tests/test_ensembles.py b/tests/test_ensembles.py index 43dc383e..54d75194 100644 --- a/tests/test_ensembles.py +++ b/tests/test_ensembles.py @@ -1046,21 +1046,18 @@ class TestEnsemblePartition: @pytest.mark.skipif(xe is None, reason="xesmf needed for testing regrdding") def test_build_partition_data(self, samplecat, tmp_path): # test subset - datasets = samplecat.search(variable="tas").to_dataset_dict( + datasets = samplecat.search(variable="tas", member="r1i1p1f1").to_dataset_dict( xarray_open_kwargs={"engine": "h5netcdf"} ) ds = xs.ensembles.build_partition_data( datasets=datasets, partition_dim=["source", "experiment"], subset_kw=dict(name="mtl", method="gridpoint", lat=[45.0], lon=[-74]), - indicators_kw=dict(indicators=[xc.atmos.tg_mean]), rename_dict={"source": "new-name"}, ) - assert ds.dims == {"time": 2, "scenario": 4, "new-name": 2} - assert ds.lat.values == 45.0 - assert ds.lon.values == -74 - assert [i for i in ds.data_vars] == ["tg_mean"] + assert ds.dims == {"time": 730, "scenario": 4, "new-name": 1} + assert ds.attrs["cat:processing_level"] == "partition-ensemble" # test regrid ds_grid = xe.util.cf_grid_2d(-75, -74, 0.25, 45, 48, 0.55) @@ -1070,6 +1067,7 @@ def test_build_partition_data(self, samplecat, tmp_path): ds = xs.ensembles.build_partition_data( datasets=datasets, regrid_kw=dict(ds_grid=ds_grid, weights_location=tmp_path), + to_level="test", ) assert ds.dims == { @@ -1080,16 +1078,43 @@ def test_build_partition_data(self, samplecat, tmp_path): "lon": 4, } assert [i for i in ds.data_vars] == ["tas"] + assert ds.attrs["cat:processing_level"] == "test" - # test error - with pytest.raises( - ValueError, - ): - ds = xs.ensembles.build_partition_data( - datasets=datasets, - subset_kw=dict(name="mtl", method="gridpoint", lat=[45.0], lon=[-74]), - indicators_kw=dict(indicators=[xc.atmos.tg_mean, xc.indicators.cf.tg]), - ) + def test_partition_from_catalog(self, samplecat): + datasets = samplecat.search(variable="tas", member="r1i1p1f1") + ds_from_dict = xs.ensembles.build_partition_data( + datasets=datasets.to_dataset_dict( + xarray_open_kwargs={"engine": "h5netcdf"} + ), + partition_dim=["source", "experiment"], + subset_kw=dict(name="mtl", method="gridpoint", lat=[45.0], lon=[-74]), + ) + + ds_from_cat = xs.ensembles.build_partition_data( + datasets=datasets, + partition_dim=["source", "experiment"], + subset_kw=dict(name="mtl", method="gridpoint", lat=[45.0], lon=[-74]), + to_dataset_kw=dict(xarray_open_kwargs={"engine": "h5netcdf"}), + ) + # fix order + ds_from_cat = ds_from_cat[["time", "model", "scenario", "tas"]] + ds_from_cat["tas"] = ds_from_cat["tas"].transpose("scenario", "model", "time") + + assert ds_from_dict.equals(ds_from_cat) + + def test_realization_partition(self, samplecat): + + datasets = samplecat.search(variable="tas").to_dataset_dict( + xarray_open_kwargs={"engine": "h5netcdf"} + ) + ds = xs.ensembles.build_partition_data( + datasets=datasets, + partition_dim=["realization", "experiment"], + subset_kw=dict(name="mtl", method="gridpoint", lat=[45.0], lon=[-74]), + ) + + assert "NCC_NorESM2-MM_r1i1p1f1" in ds.model.values + assert ds.dims == {"time": 730, "scenario": 4, "model": 2} class TestReduceEnsemble: diff --git a/tox.ini b/tox.ini index 6599f016..d382e953 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,11 @@ [tox] -min_version = 4.18.0 +min_version = 4.23.2 envlist = lint - py{310,311,312} + py{310,311,312,313} docs-esmpy requires = - pip >= 24.2.0 + pip >= 24.3.1 setuptools >= 65.0 opts = --colored @@ -13,21 +13,22 @@ opts = [gh] python = - 3.10 = py310-coveralls + 3.10 = py310-xclim-coveralls 3.11 = py311-coveralls 3.12 = py312-esmpy-coveralls + 3.13 = py313-coveralls [testenv:lint] description = Check for Code Compliance and missing french translations skip_install = True download = true deps = - black[jupyter] ==24.8.0 + black[jupyter] ==24.10.0 blackdoc ==0.3.9 isort ==5.13.2 flake8 >=7.1.1 flake8-rst-docstrings >=0.3.0 - ruff >=0.5.7 + ruff >=0.8.2 numpydoc >=1.8.0 commands_pre = pip list @@ -72,6 +73,7 @@ deps = coveralls: coveralls esmpy: git+https://github.com/esmf-org/esmf.git@v{env:ESMF_VERSION}\#subdirectory=src/addon/esmpy upstream: -rrequirements_upstream.txt + xclim: xclim=={env:XCLIM_VERSION} extras = dev install_command = python -m pip install --no-user {opts} {packages}