config-pr-1-ci.yml

name: PR Checks
on:
  workflow_call:
  # Workflows that call this workflow use the following triggers:
  # pull_request:
  #   branches:
  #     - 'release-*'
  #     - 'dev-*'
  #   paths-ignore:
  #     # These are ignored because they don't have anything to do with the model itself
  #     - .github/**
  #     - tools/**
  #     - doc/**
  #     - .*
  #     - README.md
    inputs:
      additional-artifact-content-paths:
        type: string
        required: false
        # For example, the value of 'checksum' will expand to something like
        # '/scratch/tm70/repro-ci/experiments/MODEL-configs/dev-CONFIG/checksum'
        # on the remote.
        description: |
          Newline-separated paths for inclusion in the release artifact.
          Note that all paths given have 'env.EXPERIMENT_LOCATION/' prepended.
    outputs:
      repro-ci-artifact-name:
        value: ${{ jobs.repro-ci.outputs.artifact-name }}
        description: Name of the repro-ci artifact
      repro-ci-artifact-url:
        value: ${{ jobs.repro-ci.outputs.artifact-url }}
        description: URL to the repro-ci artifact
      repro-ci-experiment-location:
        value: ${{ jobs.repro-ci.outputs.experiment-location }}
        description: Location of the experiment on the target environment
jobs:
  commit-check:
    name: Commit Check
    # We run this job to check if the current commit was done during a workflow run.
    # Such as when 'github-actions' bumps the metadata.yaml file or updates the checksums
    # in the `testing` directory.
    # This is so we don't recursively commit and check infinitely during this workflow.
    runs-on: ubuntu-latest
    outputs:
      authorship: ${{ steps.head-commit.outputs.authorship }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}

      - name: HEAD Commit Authorship
        id: head-commit
        run: echo "authorship=$(git log -1 --pretty=format:'%an')" >> $GITHUB_OUTPUT

  branch-check:
    name: PR Source Branch Check
    # This check is used as a precursor to any repro-ci checks - which are only fired
    # on dev-* -> release-* PRs.
    # This check is run to confirm that the source branch is of the form `dev-<config>`
    # and the target branch is of the form `release-<config>`. We are being especially
    # concerned with branch names because deployment to GitHub Environments can only
    # be done on source branches with a certain pattern. See ACCESS-NRI/access-om2-configs#20.
    if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && startsWith(github.base_ref, 'release-') && startsWith(github.head_ref, 'dev-')
    needs:
      - commit-check
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}

      - name: Compare Source and Target Config Names
        # In this step, we cut the 'dev-' and 'release-' to compare config names directly.
        run: |
          source=$(cut --delimiter '-' --field 2- <<< "${{ github.head_ref }}")
          target=$(cut --delimiter '-' --field 2- <<< "${{ github.base_ref }}")
          if [[ "${source}" != "${target}" ]]; then
            echo "::error::Config name of Source branch '${source}' does not match Target branch '${target}'"
            exit 1
          fi

      - name: Failure Notifier
        if: failure()
        uses: access-nri/actions/.github/actions/pr-comment@main
        with:
          comment: |
            :x: Automated Reproducibility testing cannot be run on this branch :x:
            Source and Target branches must be of the form `dev-<config>` and `release-<config>` respectively, and `<config>` must match between them.
            Rename the Source branch or check the Target branch, and try again.

  config:
    name: Read CI Testing Configuration
    runs-on: ubuntu-latest
    outputs:
      # QA Test Outputs
      qa-markers: ${{ steps.qa-config.outputs.markers }}
      qa-python-version: ${{ steps.qa-config.outputs.python-version }}
      qa-model-config-tests-version: ${{ steps.qa-config.outputs.model-config-tests-version }}
      # Repro Test Outputs
      repro-markers: ${{ steps.repro-config.outputs.markers }}
      repro-payu-version: ${{ steps.repro-config.outputs.payu-version }}
      repro-model-config-tests-version: ${{ steps.repro-config.outputs.model-config-tests-version }}
      compared-config-tag: ${{ steps.compared.outputs.tag }}
    steps:
      - name: Checkout main
        # We fetch the repository history because in `steps.compared` we
        # attempt to get the last config tag.
        uses: actions/checkout@v4
        with:
          ref: main
          fetch-depth: 0
          fetch-tags: true

      - name: Validate `config/ci.json`
        uses: access-nri/schema/.github/actions/validate-with-schema@main
        with:
          schema-version: ${{ vars.CI_JSON_SCHEMA_VERSION }}
          meta-schema-version: draft-2020-12
          schema-location: au.org.access-nri/model/configuration/ci
          data-location: config/ci.json

      - name: Read QA tests config
        id: qa-config
        uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main
        with:
          check: qa
          branch-or-tag: ${{ github.base_ref }}
          config-filepath: "config/ci.json"

      - name: Read reproducibility tests config
        id: repro-config
        uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main
        with:
          check: reproducibility
          branch-or-tag: ${{ github.base_ref }}
          config-filepath: "config/ci.json"

      - name: Get Config Tag to Compare Against
        id: compared
        # We checkout the base branch (the branch that will be merged into)
        # to get the last tagged configuration checksums.
        # We could also just take the HEAD of the base_ref, which would give
        # the same result, but tags are human readable.
        # In the case where there is no tag, `tag` will be an empty string,
        # which is handled later.
        run: |
          git checkout ${{ github.base_ref }}
          echo "tag=$(git describe --tags --abbrev=0)" >> $GITHUB_OUTPUT

  qa-ci:
    # Run quick, non-HPC tests on the runner.
    name: QA CI Checks
    needs:
      - commit-check
      - config
    if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME
    runs-on: ubuntu-latest
    permissions:
      checks: write
    steps:
      - name: Checkout PR ${{ github.event.pull_request.number }}
        uses: actions/checkout@v4

      - name: Checkout model-config-tests requirements
        # This step checks out model-config-tests/pyproject.toml because it is
        # used as a cache key for model-config-tests in the next step
        uses: actions/checkout@v4
        with:
          repository: access-nri/model-config-tests
          path: model-config-tests
          sparse-checkout: pyproject.toml
          sparse-checkout-cone-mode: false

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ needs.config.outputs.qa-python-version }}
          # We can use cache:pip to cache model-config-tests despite the actual
          # config repository not containing any python, because we checkout
          # model-config-tests/pyproject.toml, which contains data on the tests
          cache: pip
          cache-dependency-path: model-config-tests/pyproject.toml

      - name: Install model-config-tests
        run: pip install model-config-tests=='${{ needs.config.outputs.qa-model-config-tests-version }}'

      - name: Invoke Simple CI Pytests
        # We continue on error because we will let the checks generated in
        # the next step speak to the state of the testing
        continue-on-error: true
        run: |
          echo "Running pytest using '-m ${{ needs.config.outputs.qa-markers }}'"
          model-config-tests \
            -m '${{ needs.config.outputs.qa-markers }}' \
            --target-branch '${{ github.base_ref }}' \
            --junitxml=./test_report.xml

      - name: Parse Test Report
        id: tests
        uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365  #v2.12.0
        with:
          files: ./test_report.xml
          comment_mode: off
          check_run: true
          check_name: QA Test Results
          compare_to_earlier_commit: false
          report_individual_runs: true
          report_suite_logs: any

  repro-ci:
    # Run the given config on the deployment GitHub Environment (`environment-name`),
    # upload the checksums and test details, and return the result of the test.
    needs:
      - commit-check
      - branch-check
      - config
    if: needs.commit-check.outputs.authorship != vars.GH_ACTIONS_BOT_GIT_USER_NAME && needs.branch-check.result == 'success'
    uses: access-nri/model-config-tests/.github/workflows/test-repro.yml@main
    with:
      # FIXME: Make the environment name an input of some kind - what if we deploy to a different supercomputer?
      environment-name: Gadi
      config-ref: ${{ github.head_ref }}
      compared-config-ref: ${{ needs.config.outputs.compared-config-tag }}
      test-markers: ${{ needs.config.outputs.repro-markers }}
      model-config-tests-version: ${{ needs.config.outputs.repro-model-config-tests-version }}
      payu-version: ${{ needs.config.outputs.repro-payu-version }}
      additional-artifact-content-paths: ${{ inputs.additional-artifact-content-paths }}
    secrets: inherit
    permissions:
      contents: write
      checks: write

  bump-check:
    name: Version Bump Check
    # Check that the `.version` in the metadata.yaml has been modified in
    # this PR.
    needs:
      - repro-ci
    runs-on: ubuntu-latest
    steps:
      - name: Checkout PR Target
        uses: actions/checkout@v4
        with:
          ref: ${{ github.base_ref }}
          path: target

      - name: Checkout PR Source
        uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          path: source

      - name: Modification Check
        # If the PR branch version is different to the target branch (but isn't null or empty) then allow merging
        run: |
          target=$(yq e '.version' ./target/metadata.yaml)
          source=$(yq e '.version' ./source/metadata.yaml)

          echo "Comparing version target '$target' against source '$source'"

          if [[ "${source}" != "${target}" && "${source}" != "null" && "${source}" != "" ]]; then
            echo "::notice::The version has been modified to ${source}. Merging is now availible"
          else
            echo "::error::The version has not been modified in this PR. Merging is disallowed until an appropriate '!bump' is issued"
            exit 1
          fi

  result:
    name: Repro Result Notifier
    # Notify the PR of the result of the Repro check
    needs:
      - config
      - repro-ci
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    env:
      COMPARED_TAG: ${{ needs.config.outputs.compared-config-tag }}
      COMPARED_CHECKSUM_STRING: ${{ needs.config.outputs.compared-config-tag != '' && format('The checksums compared against are found here {0}/{1}/tree/{2}/testing/checksum', github.server_url, github.repository, needs.config.outputs.compared-config-tag) || '' }}
    steps:
      - name: Successful Repro Comment
        if: needs.repro-ci.outputs.result == 'pass'
        uses: access-nri/actions/.github/actions/pr-comment@main
        with:
          comment: |
            :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ env.COMPARED_TAG }}` :white_check_mark:
            You must bump the minor version of this configuration - to bump the version, comment `!bump minor` or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.

            <details>
            <summary> Further information</summary>

            The experiment can be found on Gadi at `${{ needs.repro-ci.outputs.experiment-location }}`, and the test results at ${{ needs.repro-ci.outputs.check-run-url }}.

            The checksums generated in this PR are found in the `testing/checksum` directory of ${{ needs.repro-ci.outputs.artifact-url }}.

            ${{ env.COMPARED_CHECKSUM_STRING }}

            </details>

      - name: Failed Repro Comment
        if: needs.repro-ci.outputs.result == 'fail'
        uses: access-nri/actions/.github/actions/pr-comment@main
        with:
          comment: |
            :x: The Bitwise Reproducibility check failed ${{ env.COMPARED_TAG != '' && format('when comparing against `{0}`', env.COMPARED_TAG) || 'as there is no earlier checksum to compare against' }} :x:
            You must bump the major version of this configuration - to bump the version, comment `!bump major`or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.

            <details>
            <summary> Further information</summary>

            The experiment can be found on Gadi at `${{ needs.repro-ci.outputs.experiment-location }}`, and the test results at ${{ needs.repro-ci.outputs.check-run-url }}.

            The checksums generated in this PR are found in the `testing/checksum` directory of ${{ needs.repro-ci.outputs.artifact-url }}.

            ${{ env.COMPARED_CHECKSUM_STRING }}

            </details>