Model Repro CI

.github/workflows/pr-1-ci.yml

@@ -0,0 +1,153 @@
+name: PR Checks
+on:
+  pull_request:
+    branches:
+      - 'release-*'
+      - 'dev-*'
+    paths-ignore:
+      # These are ignored because they don't have anything to do with the model itself
+      - .github/**
+      - tools/**
+      - doc/**
+      - .*
+      - metadata.yaml
+      - README.md
+jobs:
+  commit-check:
+    name: Commit Check
+    # We run this job to check if the current commit was done during a workflow run.
+    # Such as when 'github-actions' bumps the metadata.yaml file or updates the checksums
+    # in the `testing` directory.
+    # This is so we don't recursively commit and check infinitely during this workflow.
+    runs-on: ubuntu-latest
+    outputs:
+      authorship: ${{ steps.head-commit.outputs.authorship }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: HEAD Commit Authorship
+        id: head-commit
+        run: echo "authorship=$(git log -1 --pretty=format:'%an')" >> $GITHUB_OUTPUT
+
+  repro-ci:
+    # run the given config on the deployment GitHub Environment (`environment-name`) and
+    # upload the checksums and test details
+    needs:
+      - commit-check
+    if: needs.commit-check.outputs.authorship != 'github-actions'
+    uses: access-nri/reproducibility/.github/workflows/checks.yml@main
+    with:
+      model-name: access-om2
+      environment-name: Gadi
+      config-tag: ${{ github.head_ref }}
+      test-markers: checksum
+    secrets: inherit
+    permissions:
+      contents: write
+
+  check-checksum:
+    # Parse the test report and return pass/fail result
+    name: Check and Update Checksum
+    needs:
+      - repro-ci
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+      checks: write
+    env:
+      TESTING_LOCAL_LOCATION: /opt/testing
+    outputs:
+      # URL for the parsed test results
+      check-run-url: ${{ steps.results.outputs.check-url }}
+      # Overall result of the checksum repro CI - `pass` (if reproducible), `fail` otherwise
+      result: ${{ steps.results.outputs.result }}
+      # Version of the checksum compared against the newly generated one
+      compared-checksum-version: ${{ steps.results.outputs.compared-checksum-version }}
+    steps:
+      - name: Download Newly Created Checksum
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ needs.repro-ci.outputs.artifact-name }}
+          path: ${{ env.TESTING_LOCAL_LOCATION }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Parse Test Report
+        id: tests
+        uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365  #v2.12.0
+        with:
+          files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml
+          comment_mode: off
+          check_run: true
+          compare_to_earlier_commit: false
+          report_individual_runs: true
+          report_suite_logs: any
+
+      - name: Checksum Tests Results
+        id: results
+        run: |
+          echo "check-url=${{ fromJson(steps.tests.outputs.json).check_url }}" >> $GITHUB_OUTPUT
+          echo "compared-checksum-version=$(git describe --tags --abbrev=0)" >> $GITHUB_OUTPUT
+          if [ "${{ fromJson(steps.tests.outputs.json).stats.tests_fail }}" > 0 ]; then
+            echo "result=fail" >> $GITHUB_OUTPUT
+          else
+            echo "result=pass" >> $GITHUB_OUTPUT
+          fi
+
+  result:
+    name: Repro Result Notifier
+    # Notify the PR of the result of the Repro check
+    needs:
+      - repro-ci
+      - check-checksum
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_REPO: ${{ github.repository }}
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Successful Release Comment
+        if: needs.check-checksum.outputs.result == 'pass' && startsWith(github.base_ref, 'release-')
+        env:
+          BODY: |
+            :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Release` config. :white_check_mark:
+            For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}.
+            Consider bumping the minor version of `access-om2-configs` - to bump the version, comment `!bump minor`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.
+        run: gh pr comment --body '${{ env.BODY }}'
+
+      - name: Successful Dev Comment
+        if: needs.check-checksum.outputs.result == 'pass' && startsWith(github.base_ref, 'dev-')
+        env:
+          BODY: |
+            :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Dev` config. :white_check_mark:
+            For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}.
+        run: gh pr comment --body '${{ env.BODY }}'
+
+      - name: Failed Release Comment
+        if: needs.check-checksum.outputs.result == 'fail' && startsWith(github.base_ref, 'release-')
+        env:
+          BODY: |
+            :x: The Bitwise Reproducibility check failed when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Release` config. :x:
+            For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}.
+            You must bump the major version of `access-om2-configs` before this PR is merged to account for this - to bump the version, comment `!bump major`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.
+        run: gh pr comment --body '${{ env.BODY }}'
+
+      - name: Failed Dev Comment
+        if: needs.check-checksum.outputs.result == 'fail' && startsWith(github.base_ref, 'dev-')
+        env:
+          BODY: |
+            :warning: The Bitwise Reproducibility check failed when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` for this `Dev` config. :warning:
+            For further information, the experiment can be found on Gadi at ${{ needs.repro-ci.outputs.experiment-location }}, and the test results at ${{ needs.check-checksum.outputs.check-run-url }}.
+        run: gh pr comment --body '${{ env.BODY }}'

.github/workflows/pr-2-confirm.yml

@@ -0,0 +1,160 @@
+# This workflow is used to do a major/minor version bump to the `metadata.yaml` file,
+# through a comment on the PR. It also commits and pushes the checksum file,
+# as this is the last stage before merging.
+# This is not done automatically because users may want to modify their config
+# based on the result of the reproducibility check.
+name: Confirm
+on:
+  issue_comment:
+    types:
+      - created
+      - edited
+env:
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+jobs:
+  bump-version:
+    name: Bump metadata.yaml
+    # Bump the `metadata.yaml` file if the comment is made on a PR and starts with '!bump'
+    if: github.event.issue.pull_request && startsWith(github.event.comment.body, '!bump')
+    runs-on: ubuntu-latest
+    outputs:
+      # metadata.yaml version before being bumped
+      before: ${{ steps.bump.outputs.before }}
+      # metadata.yaml version after being bumped
+      after: ${{ steps.bump.outputs.after }}
+      # The type of bump - 'major' or 'minor'
+      type: ${{ steps.type.outputs.bump }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }}
+
+      - name: Checkout Associated PR ${{ github.event.issue.number }}
+        # Since the trigger for this workflow was on.issue_comment, we need
+        # to do a bit more wrangling to checkout the pull request
+        id: pr
+        run: gh pr checkout ${{ github.event.issue.number }}
+
+      - name: Get Type of Bump
+        id: type
+        run: |
+          if [[ "${{ contains(github.event.comment.body, 'minor') }}" == "true" ]]; then
+            echo "bump=minor" >> $GITHUB_OUTPUT
+          elif [[ ${{ contains(github.event.comment.body, 'major') }} == "true" ]]; then
+            echo "bump=major" >> $GITHUB_OUTPUT
+          else
+            echo "::error::Comment was not of the form: '!bump [major|minor]'"
+            exit 1
+          fi
+
+      - name: Bump
+        # Regarding the regex in the script: `([0-9]+)\.([0-9]+)` is broken down into:
+        # `([0-9]+)`: Major version (eg. `12`)
+        # `\.`: Version separator (eg. `.`)
+        # `([0-9]+)`: Minor version (eg. `1`)
+        # which would give `12.1`
+        id: bump
+        run: |
+          version=$(yq '.version' metadata.yaml)
+          regex="([0-9]+)\.([0-9]+)"
+          if [[ $version =~ $regex ]]; then
+            major_version="${BASH_REMATCH[1]}"
+            minor_version="${BASH_REMATCH[2]}"
+          else
+            echo "::error::Invalid version format in metadata.yaml file!"
+            exit 1
+          fi
+
+          if [[ "${{ steps.type.outputs.bump }}" == "minor" ]]; then
+            minor_version=$((minor_version + 1))
+          elif [[ "${{ steps.type.outputs.bump }}" == "major" ]]; then
+            major_version=$((major_version + 1))
+            minor_version=0
+          fi
+          new_version="${major_version}.${minor_version}"
+          echo "before=$version" >> $GITHUB_OUTPUT
+          echo "after=$new_version" >> $GITHUB_OUTPUT
+
+  commit:
+    name: Commit metadata.yaml and Checksum
+    needs:
+      - bump-version
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      GH_TOKEN: ${{ secrets.GH_COMMIT_CHECK_TOKEN }}
+      ARTIFACT_LOCAL_LOCATION: /opt/artifact
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }}
+
+      - name: Checkout Associated PR ${{ github.event.issue.number }}
+        # Since the trigger for this workflow was on.issue_comment, we need
+        # to do a bit more wrangling to checkout the pull request and get the branch name
+        id: pr
+        run: |
+          gh pr checkout ${{ github.event.issue.number }}
+          echo "branch=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Download Newly Created Checksum
+        # Given the PR branch, we need to find the latest associated workflow run
+        # on this branch we can then download the associated artifact
+        run: |
+          associated_run=$(gh run list \
+            --json='databaseId,headBranch,updatedAt,status' \
+            --jq='[.[] | select(.headBranch == "${{ steps.pr.outputs.branch }}" and .status == "completed")] | sort_by(.updatedAt) | last | .databaseId')
+          gh run download $associated_run -D ${{ env.ARTIFACT_LOCAL_LOCATION }}
+
+      - name: Update metadata.yaml and Checksum files
+        run: |
+          yq -i '.version = "${{ needs.bump-version.outputs.after }}"' metadata.yaml
+          cp --recursive --verbose ${{ env.ARTIFACT_LOCAL_LOCATION }}/*/* testing
+
+      - name: Commit and Push Updates
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+
+          if [[ "${{ needs.bump-version.outputs.type }}" == "minor" ]]; then
+            git commit -am "Bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}"
+          elif [[ "${{ needs.bump-version.outputs.type }}" == "major" ]]; then
+            git commit -am "Updated checksums and bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}"
+          fi
+          git push
+
+      - name: Comment Success
+        env:
+          BODY: |
+            :white_check_mark: Version bumped from `${{ needs.bump-version.outputs.before }}` to `${{ needs.bump-version.outputs.after }}` :white_check_mark:
+        run: gh pr comment --body '${{ env.BODY }}'
+
+  failure-notifier:
+    name: Failure Notifier
+    if: failure()
+    needs:
+      - commit
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_REPO: ${{ github.repository }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Checkout Associated PR ${{ github.event.issue.number }}
+        run: gh pr checkout ${{ github.event.issue.number }}
+
+      - name: Comment Failure
+        env:
+          BODY: |
+            :x: Failed to bump VERSION or commit changes, see ${{ env.RUN_URL }} :x:
+        run: gh pr comment --body '${{ env.BODY }}'

.github/workflows/pr-3-bump-tag.yml

@@ -0,0 +1,41 @@
+# This workflow is used to convert the `.version` in the `metadata.yaml` file into a valid `git tag` on push to `main`.
+# We use the `.version` field in that file to denote the version of the config once a PR is merged.
+name: Bump Tag
+on:
+  push:
+    branches:
+      - 'release-*'
+    paths:
+      - 'metadata.yaml'
+jobs:
+  tag-update:
+    name: Check and Update Tag
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+
+      - name: Existing Tag Check
+        # Check if the tag already exists, if it does, we don't want to move it.
+        id: tag
+        run: |
+          VERSION_TAG=${{ github.ref_name }}-$(yq '.version' metadata.yaml)
+          VERSION_TAG_ON_GIT=$(git tag -l $VERSION_TAG)
+          if [ -n "$VERSION_TAG_ON_GIT" ]; then
+            echo "::warning::Tag $VERSION_TAG already exists. Skipping."
+            echo "exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "exists=false" >> $GITHUB_OUTPUT
+            echo "version=$VERSION_TAG" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Update Tag
+        if: steps.tag.outputs.exists == 'false'
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git tag ${{ steps.tag.outputs.version }}
+          git push --tags

.github/workflows/schedule-1-ci.yml

@@ -0,0 +1,36 @@
+name: Scheduled Checks
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 1 * *'  # once a month
+jobs:
+  setup:
+    name: Setup Tasks
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.get-released-config.outputs.tags }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Get all released configs
+        id: get-released-config
+        run: echo "tags=$(jq --compact-output --raw-output '.tags' config/released-configs.json)" >> $GITHUB_OUTPUT
+
+  repro-ci:
+    # We use this reusable workflow with a matrix strategy rather than calling repro-ci.yml, as
+    # we may want to do config-branch-specific tasks after the matrixed repro-ci.yml has completed.
+    needs:
+      - setup
+    strategy:
+      fail-fast: false
+      matrix:
+        config-tag: ${{ fromJson(needs.setup.outputs.tags) }}
+    uses: ./.github/workflows/schedule-2-start.yml
+    with:
+      config-tag: ${{ matrix.config-tag }}
+    secrets: inherit
+    permissions:
+      contents: write
+      issues: write