Merge pull request #4 from ACCESS-NRI/2-ci-cd

Add CI/CD to ACCESS-OM3

.github/workflows/cd.yml

@@ -0,0 +1,47 @@
+name: CD
+on:
+  push:
+    branches:
+      - main
+      - backport/*.*
+    paths:
+      - config/**
+      - spack.yaml
+env:
+  SPACK_YAML_MODEL_YQ: .spack.specs[0]
+jobs:
+  push-tag:
+    name: Tag Deployment
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    outputs:
+      name: ${{ steps.tag.outputs.name }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate Tag
+        id: tag
+        # Get the tag name from the `spack.yaml` that was merged into main, which
+        # is of the form `access-om3@git.<version>`.
+        run: echo "name=$(yq '${{ env.SPACK_YAML_MODEL_YQ }} | split("@git.") | .[1]' spack.yaml)" >> $GITHUB_OUTPUT
+
+      - name: Push Tag
+        # NOTE: Regarding the config user.name/user.email, see https://github.com/actions/checkout/pull/1184
+        run: |
+          git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }}
+          git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }}
+          git tag ${{ steps.tag.outputs.name }} --force
+          git push --tags --force
+
+  deploy-release:
+    name: Deploy Release
+    needs:
+      - push-tag
+    uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main
+    with:
+      ref: ${{ github.ref_name }}
+      version: ${{ needs.push-tag.outputs.name }}
+    secrets: inherit
+    permissions:
+      contents: write

.github/workflows/ci.yml

@@ -0,0 +1,178 @@
+name: CI
+on:
+  pull_request:
+    branches:
+      - main
+      - backport/*.*
+    paths:
+      - config/**
+      - spack.yaml
+env:
+  SPACK_YAML_MODEL_YQ: .spack.specs[0]
+jobs:
+  validate-json:
+    name: Validate JSON
+    uses: access-nri/actions/.github/workflows/validate-json.yml@main
+    with:
+      src: "config"
+
+  check-config:
+    name: Check Config Fields
+    needs:
+      - validate-json
+    runs-on: ubuntu-latest
+    outputs:
+      spack-packages-version: ${{ steps.spack-packages.outputs.version }}
+      spack-config-version:  ${{ steps.spack-config.outputs.version }}
+    steps:
+      - name: Validate spack-packages version
+        id: spack-packages
+        uses: access-nri/build-cd/.github/actions/validate-repo-version@main
+        with:
+          repo-to-check: spack-packages
+
+      - name: Validate spack-config version
+        id: spack-config
+        uses: access-nri/build-cd/.github/actions/validate-repo-version@main
+        with:
+          repo-to-check: spack-config
+
+  check-spack-yaml:
+    name: Check spack.yaml
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check Model Version Modified
+        id: version
+        run: |
+          git checkout ${{ github.base_ref }}
+          base_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml)
+
+          git checkout ${{ github.head_ref }}
+          current_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml)
+          echo "current=${current_version}" >> $GITHUB_OUTPUT
+
+          if [[ "${base_version}" == "${current_version}" ]]; then
+            echo "::warning::The version string hasn't been modified in this PR, but needs to be before merging."
+            exit 1
+          fi
+
+      - name: Same Model Version Failure Notifier
+        if: failure() && steps.version.outcome == 'failure'
+        uses: access-nri/actions/.github/actions/pr-comment@main
+        with:
+          comment: |
+            The model version in the `spack.yaml` has not been updated.
+            Either update it manually, or comment the following to have it updated and committed automatically:
+            * `!bump major` for feature releases
+            * `!bump minor` for bugfixes
+
+      - name: Projection Version Matches
+        # this step checks that the versions of the packages themselves match with the
+        #  names of the modules. For example, [email protected] matches with the
+        #  modulefile access-om3/2023.12.12 (specifically, the version strings match)
+        run: |
+          FAILED='false'
+          DEPS=$(yq ".spack.modules.default.tcl.include | join(\" \")" spack.yaml)
+
+          # for each of the modules
+          for DEP in $DEPS; do
+            DEP_VER=''
+            if [[ "$DEP" == "access-om3-virtual" ]]; then
+              DEP_VER=$(yq '.spack.specs[0] | split("@git.") | .[1]' spack.yaml)
+            else
+              DEP_VER=$(yq ".spack.packages.\"$DEP\".require[0] | split(\"@git.\") | .[1]" spack.yaml)
+            fi
+
+            MODULE_VER=$(yq ".spack.modules.default.tcl.projections.\"$DEP\" | split(\"/\") | .[1]" spack.yaml)
+
+            if [[ "$DEP_VER" != "$MODULE_VER" ]]; then
+              echo "::error::Version of dependency and projection do not match ($DEP_VER != $MODULE_VER)"
+              FAILED='true'
+            fi
+          done
+          if [[ "$FAILED" == "true" ]]; then
+            exit 1
+          fi
+
+  version-tag:
+    name: Get Version and Tag Prerelease
+    needs:
+      - check-spack-yaml
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    outputs:
+      release: ${{ steps.version.outputs.release }}
+      prerelease: ${{ steps.version.outputs.prerelease }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+
+      - name: Generate Versions
+        id: version
+        # This step generates the release and prerelease version numbers.
+        # The release is a general version number from the spack.yaml, looking the
+        # same as a regular release build. Ex. '[email protected]' -> '2024.01.1'
+        # The prerelease looks like: `pr<pull request number>-<number of commits on this branch>`.
+        # Ex. Pull Request #12 with 2 commits on branch -> `pr12-2`.
+        run: |
+          echo "release=$(yq '${{ env.SPACK_YAML_MODEL_YQ }} | split("@git.") | .[1]' spack.yaml)" >> $GITHUB_OUTPUT
+
+          number_of_commits=$(git rev-list --count ${{ github.event.pull_request.base.sha }}..HEAD)
+          echo "prerelease=pr${{ github.event.pull_request.number }}-${number_of_commits}" >> $GITHUB_OUTPUT
+
+      - name: Shift Prerelease Tag ${{ steps.version.outputs.release }}
+        # We shift the 'Release' tag along the PR as the spack.yaml will not work without the correct tag in this repostiory.
+        # NOTE: Regarding the config user.name/user.email, see https://github.com/actions/checkout/pull/1184
+        run: |
+          git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }}
+          git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }}
+          git tag ${{ steps.version.outputs.release }} --force
+          git push --tags --force
+
+  # -----------------------------
+  # | PRERELEASE DEPLOYMENT JOB |
+  # -----------------------------
+  prerelease-deploy:
+    name: Deploy to Prerelease
+    # This will create a `spack` environment with the name `access-om3-pr<pull request number>-<commit number>`.
+    # For example, `access-om3-pr13-3` for the deployment based on the third commit on the PR#13.
+    needs:
+      - version-tag  # implies all the spack.yaml-related checks have passed, has appropriate version for the prerelease build
+      - check-config  # implies all the json-related checks have passed
+    uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main
+    with:
+      type: prerelease
+      ref: ${{ github.head_ref }}
+      version: ${{ needs.version-tag.outputs.prerelease }}
+    secrets: inherit
+
+  notifier:
+    name: Notifier
+    needs:
+      - version-tag  # implies all the spack.yaml-related checks have passed, has appropriate version for the prerelease build
+      - check-config  # implies all the json-related checks have passed
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: access-nri/actions/.github/actions/pr-comment@main
+        with:
+         comment: |
+            This `${{ github.repository }}` model will be deployed with the following versions:
+            * `${{ needs.version-tag.outputs.release }}` as a Release (when merged).
+            * `${{ needs.version-tag.outputs.prerelease }}` as a Prerelease (during this PR). This can be accessed on `Gadi` via `spack` at `/g/data/vk83/prerelease/apps/spack/0.20/spack` once deployed.
+
+            It will be deployed using:
+            * `access-nri/spack-packages` version [`${{ needs.check-config.outputs.spack-packages-version }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.check-config.outputs.spack-packages-version }})
+            * `access-nri/spack-config` version [`${{ needs.check-config.outputs.spack-config-version }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.check-config.outputs.spack-config-version }})
+
+            If this is not what was expected, commit changes to `config/versions.json`.

.github/workflows/comment.yml

@@ -0,0 +1,99 @@
+name: Comment Command
+on:
+  issue_comment:
+    types:
+      - created
+      - edited
+env:
+  RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+  SPACK_YAML_MODEL_YQ: .spack.specs[0]
+  SPACK_YAML_MODEL_PROJECTION_YQ: .spack.modules.default.tcl.projections.access-om3-virtual
+jobs:
+  bump-version:
+    name: Bump spack.yaml
+    if: github.event.issue.pull_request && startsWith(github.event.comment.body, '!bump')
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+          token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }}
+
+      - name: Setup
+        id: setup
+        # outputs:
+        #   original-version: The version contained within the spack.yaml
+        #   version: The version that will be bumped (could be latest tag instead of original-version)
+        #   bump: The bump type (major, minor or current as specified in the bump-version action)
+        run: |
+          # Get the version of access-om3-virtual from the spack.yaml in the PR the comment was written in
+          gh pr checkout ${{ github.event.issue.number }}
+          original_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }} | split("@git.") | .[1]' spack.yaml)
+          echo "original-version=${original_version}" >> $GITHUB_OUTPUT
+
+          # Validate the comment
+          if [[ "${{ contains(github.event.comment.body, 'major') }}" == "true" ]]; then
+            # Compare the current date (year-month) with the latest git tag (year-month)
+            # to determine the next valid tag. We do this because especially feature-rich
+            # months might increment the date part beyond the current date.
+
+            d="$(date +%Y-%m)-01"
+            d_s=$(date --date "$d" +%s)
+
+            latest_tag=$(git describe --tags --abbrev=0 | tr '.' '-')
+            tag_date=${latest_tag%-*}-01
+            tag_date_s=$(date --date "$tag_date" +%s)
+
+            echo "Comparing current date ${d} with ${tag_date} (tag looks like ${latest_tag})"
+
+            if (( d_s <= tag_date_s )); then
+              echo "version=${tag_date}" >> $GITHUB_OUTPUT
+              echo "bump=major" >> $GITHUB_OUTPUT
+            else
+              echo "version=${original_version}" >> $GITHUB_OUTPUT
+              echo "bump=current" >> $GITHUB_OUTPUT
+            fi
+          elif [[ "${{ contains(github.event.comment.body, 'minor')}}" == "true" ]]; then
+            echo "version=${original_version}" >> $GITHUB_OUTPUT
+            echo "bump=minor" >> $GITHUB_OUTPUT
+          else
+            echo "::warning::Usage: `!bump [major|minor]`, got `${{ github.event.comment.body }}`"
+            exit 1
+          fi
+
+      - name: Bump Version
+        id: bump
+        uses: access-nri/actions/.github/actions/bump-version@main
+        with:
+          version: ${{ steps.setup.outputs.version }}
+          versioning-scheme: calver-minor
+          bump-type: ${{ steps.setup.outputs.bump }}
+
+      - name: Update, Commit and Push the Bump
+        run: |
+          git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }}
+          git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }}
+
+          yq -i '${{ env.SPACK_YAML_MODEL_YQ }} = "access-om3-virtual@git.${{ steps.bump.outputs.after }}"' spack.yaml
+          yq -i '${{ env.SPACK_YAML_MODEL_PROJECTION_YQ }} = "{name}/${{ steps.bump.outputs.after }}"' spack.yaml
+          git add spack.yaml
+          git commit -m "spack.yaml: Updated access-om3-virtual package version from ${{ steps.setup.outputs.original-version }} to ${{ steps.bump.outputs.after }}"
+          git push
+
+      - name: Success Notifier
+        uses: access-nri/actions/.github/actions/pr-comment@main
+        with:
+          comment: |
+            :white_check_mark: Version bumped from `${{ steps.setup.outputs.original-version }}` to `${{ steps.bump.outputs.after }}` :white_check_mark:
+
+      - name: Failure Notifier
+        if: failure()
+        uses: access-nri/actions/.github/actions/pr-comment@main
+        with:
+          comment: |
+            :x: Failed to bump version or commit changes, see ${{ env.RUN_URL }} :x:

.github/workflows/pr-closed.yml

@@ -0,0 +1,36 @@
+name: PR Closed Cleanup
+# Remove prereleases that were part of a closed PR, so we save space
+# on our deployment targets. If needed, one can still get the
+# spack.yaml as part of the closed PR and revive it themselves.
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+      - backport/*.*
+    paths:
+      - config/**
+      - spack.yaml
+jobs:
+  setup:
+    name: Setup
+    runs-on: ubuntu-latest
+    outputs:
+      version-pattern: ${{ steps.version.outputs.pattern }}
+    steps:
+      - name: Version Pattern
+        id: version
+        # For example, `access-om3-pr12-*`
+        run: |
+          repo_name_lower=$(echo ${{ github.event.repository.name }} | tr [:upper:] [:lower:])
+          echo "pattern=${repo_name_lower}-pr${{ github.event.pull_request.number }}-*" >> $GITHUB_OUTPUT
+
+  undeploy-prereleases:
+    name: Undeploy Prereleases
+    needs:
+      - setup
+    uses: access-nri/build-cd/.github/workflows/undeploy-1-setup.yml@main
+    with:
+      version-pattern: ${{ needs.setup.outputs.version-pattern }}
+    secrets: inherit