From cac6a394d711b44b1ed9e0d85cc568f04338dec7 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 17 Oct 2024 16:23:43 +1100 Subject: [PATCH] `!test` Comment Command (#73) * Add config-comment-repro.yml workflow for repro tests on any branch! * Replace `against COMMITISH` with `compare COMMITISH` * Added ability to commit the result of the repro test to the PR * Added RUN_URL, removed some of the branch name stuff * Add additional artifact content paths section for repro-ci * Removed custom `[compare COMMITISH]` logic * Update artifact structure to be in line with https://github.com/ACCESS-NRI/model-config-tests/pull/74 * Renamed file, made it a more generic `!test` command * Changed artifact paths back to initial version, removed `inputs.additional-artifact-content-paths` input * Removed issue property from `github.event.comment.body` * Reworked permission checking, bug fixes, comment updates * Further changes from testing * Added error-checking for `!test` command * Made comments more informative * Update README.md to account for new `!test repro` syntax * Update .github/workflows/config-comment-test.yml Co-authored-by: Aidan Heerdegen * README.md: Note the case-sensitivity of the command * Add step in prepare-command to check the command starts with `!test` --------- Co-authored-by: Aidan Heerdegen --- .github/workflows/config-comment-test.yml | 375 ++++++++++++++++++++++ README.md | 45 ++- 2 files changed, 413 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/config-comment-test.yml diff --git a/.github/workflows/config-comment-test.yml b/.github/workflows/config-comment-test.yml new file mode 100644 index 0000000..33268b9 --- /dev/null +++ b/.github/workflows/config-comment-test.yml @@ -0,0 +1,375 @@ +name: 'Comment Command: !test' +run-name: '!test on ${{ github.repository }}' +on: + workflow_call: + # Triggered in calling workflow by: + # on: + # issue_comment: + # - edited + # - created +env: + USAGE: '!test TYPE [commit]' + USAGE_TYPES: repro + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} +jobs: + permission-check: + name: Permission Check + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + permissions: + pull-requests: write + steps: + - name: Determine if commenter has permission to run the command + id: commenter + # TODO: The permissions-checking section seems ripe for turning into an action! + run: | + commenter_permissions=$(gh api \ + repos/${{ github.repository }}/collaborators/${{ github.event.comment.user.login }}/permission \ + --jq '.permission' + ) + if [[ "$commenter_permissions" == "admin" || "$commenter_permissions" == "write" ]]; then + echo "Commenter has at least write permission" + echo "permission=true" >> $GITHUB_OUTPUT + else + echo "Commenter does not have at least write permission" + echo "permission=false" >> $GITHUB_OUTPUT + fi + + - name: React to '!test' + uses: access-nri/actions/.github/actions/react-to-comment@main + with: + reaction: ${{ steps.commenter.outputs.permission == 'true' && 'rocket' || 'confused' }} + token: ${{ github.token }} + + - name: Fail if no permissions + if: steps.commenter.outputs.permission == 'false' + run: exit 1 + + ci-config: + name: Read CI Testing Configuration + needs: + - permission-check + runs-on: ubuntu-latest + outputs: + markers: ${{ steps.repro-config.outputs.markers }} + payu-version: ${{ steps.repro-config.outputs.payu-version }} + model-config-tests-version: ${{ steps.repro-config.outputs.model-config-tests-version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + + - name: Validate `config/ci.json` + uses: access-nri/schema/.github/actions/validate-with-schema@main + with: + # As with a lot of the `vars`/`secrets` in this repo, this will be defined in the caller repo + schema-version: ${{ vars.CI_JSON_SCHEMA_VERSION }} + meta-schema-version: draft-2020-12 + schema-location: au.org.access-nri/model/configuration/ci + data-location: config/ci.json + + - name: Get base branch for PR + id: base + env: + GH_TOKEN: ${{ github.token }} + run: | + gh pr checkout ${{ github.event.issue.number }} + echo "branch=$(gh pr view --json baseRefName --jq '.baseRefName')" + + - name: Read reproducibility tests config + id: repro-config + uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main + with: + check: reproducibility + branch-or-tag: ${{ steps.base.outputs.branch }} + config-filepath: "config/ci.json" + + prepare-command: + name: Prepare Command + needs: + - permission-check + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + outputs: + test-type: ${{ steps.test.outputs.type }} + # the full git hash of the configuration being tested + config-hash: ${{ steps.pr.outputs.hash }} + # the short git hash of the configuration being tested + config-short-hash: ${{ steps.pr.outputs.short-hash }} + # the git ref (branch or tag) of the configuration being tested + config-ref: ${{ steps.pr.outputs.ref }} + # the full git hash of the configuration being compared against + compared-config-hash: ${{ steps.compared.outputs.hash }} + # the short git hash of the configuration being compared against + compared-config-short-hash: ${{ steps.compared.outputs.short-hash }} + # the git ref (branch or tag) of the configuration being compared against + compared-config-ref: ${{ steps.compared.outputs.ref }} + # whether the commenter can commit to the repo (either 'true', 'false' or '' (when no 'commit' option given)) + commit-requested: ${{ steps.commit.outputs.requested }} + # TODO: Make this an input to the command when we start deploying to multiple targets + environment-name: Gadi + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Verify !test + run: | + if [[ "${{ startsWith(github.event.comment.body, '!test') }}" == "true" ]]; then + echo 'Command starts with !test' + else + echo "::error::Usage: ${{ env.USAGE }}" + echo "::error::Command must start with !test to invoke model-config-tests' config-comment-test.yml." + exit 1 + + - name: Get !test type + id: test + run: | + command="${{ github.event.comment.body }}" + read -ra command_tokens <<< "$command" + type_in_comment="${command_tokens[1]}" + type_in_comment_valid=false + + for type in ${{ env.USAGE_TYPES }}; do + if [[ "$type_in_comment" == "$type" ]]; then + type_in_comment_valid=true + break + fi + done + + if [[ "$type_in_comment_valid" == "false" ]]; then + echo "::error::Usage: ${{ env.USAGE }}" + echo "::error::The command '${{ github.event.comment.body }}' doesn't have a valid TYPE. Was given '$type_in_comment', but needed to be one of: ${{ env.USAGE_TYPES }}." + exit 1 + fi + + echo "type=$type_in_comment" >> $GITHUB_OUTPUT + + - name: Get config ref from PR comment + id: pr + run: | + gh pr checkout ${{ github.event.issue.number }} + echo "hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "short-hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "ref=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT + + - name: Get compared config ref + id: compared + # Set up a default ref + # We need to do this roundabout way to get the source branch commit because + # We can't access github.base_ref as this trigger is on.issue_comment + run: | + ref=$(gh pr view ${{ github.event.issue.number }} --json baseRefName --jq '.baseRefName') + echo "Using $ref" + hash=$(git show-branch --merge-base origin/$ref origin/${{ steps.pr.outputs.ref }}) + short_hash=$(git rev-parse --short $hash) + echo "AKA $hash (shortened as $short_hash)" + + echo "hash=$hash" >> $GITHUB_OUTPUT + echo "short-hash=$short_hash" >> $GITHUB_OUTPUT + echo "ref=$ref" >> $GITHUB_OUTPUT + + - name: Determine whether to commit + id: commit + # Determine whether the commenter wants to commit something to the repository + run: | + command="${{ github.event.comment.body }}" + read -ra command_tokens <<< "$command" + potential_commit_token="${command_tokens[2]}" + + if [[ "$potential_commit_token" == "commit" ]]; then + echo "'commit' option given" + echo "requested=true" >> $GITHUB_OUTPUT + elif [ -z "$potential_commit_token" ]; then + echo "'commit' option not given" + echo "requested=false" >> $GITHUB_OUTPUT + else + echo "::error::Non-commit option '$potential_commit_token' given. Usage: ${{ env.USAGE }}" + exit 1 + fi + + - name: Erroneous tokens check + # If commit is given, any tokens after the 3rd are considered erroneous - otherwise it is after the 2nd + # TODO: This will need to be made more robust if there are other options added + run: | + command="${{ github.event.comment.body }}" + read -ra command_tokens <<< "$command" + + max_tokens_allowed=$([[ "${{ steps.commit.outputs.requested }}" == "true" ]] && echo "3" || echo "2") + + if [ "${#command_tokens[@]}" -gt "$max_tokens_allowed" ]; then + echo "::error::Erroneous tokens given. Usage: ${{ env.USAGE }}" + exit 1 + fi + + repro: + name: Compare ${{ needs.prepare-command.outputs.config-ref }} against ${{ needs.prepare-command.outputs.compared-config-ref }} + needs: + - prepare-command + - ci-config + if: needs.prepare-command.outputs.test-type == 'repro' + uses: access-nri/model-config-tests/.github/workflows/test-repro.yml@main + with: + config-ref: ${{ needs.prepare-command.outputs.config-hash }} + compared-config-ref: ${{ needs.prepare-command.outputs.compared-config-hash }} + environment-name: ${{ needs.prepare-command.outputs.environment-name }} + payu-version: ${{ needs.ci-config.outputs.payu-version }} + model-config-tests-version: ${{ needs.ci-config.outputs.model-config-tests-version }} + test-markers: ${{ needs.ci-config.outputs.markers }} + secrets: inherit + permissions: + contents: write + + check-repro: + # Parse the test report and return pass/fail result + name: Results + needs: + - prepare-command + - repro + runs-on: ubuntu-latest + env: + TESTING_LOCAL_LOCATION: /opt/testing + permissions: + pull-requests: write + checks: write + outputs: + result: ${{ steps.results.outputs.result }} + steps: + - name: Download Newly Created Checksum + uses: actions/download-artifact@v4 + with: + name: ${{ needs.repro.outputs.artifact-name }} + path: ${{ env.TESTING_LOCAL_LOCATION }} + + - name: Parse Test Report + id: tests + uses: EnricoMi/publish-unit-test-result-action/composite@82082dac68ad6a19d980f8ce817e108b9f496c2a #v2.17.1 + with: + files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml + comment_mode: off + check_run: true + compare_to_earlier_commit: false + report_individual_runs: true + report_suite_logs: any + + - name: Repro results + id: results + run: | + echo "check-url=${{ fromJson(steps.tests.outputs.json).check_url }}" >> $GITHUB_OUTPUT + + if (( ${{ fromJson(steps.tests.outputs.json).stats.tests_fail }} > 0 )); then + echo "result=fail" >> $GITHUB_OUTPUT + else + echo "result=pass" >> $GITHUB_OUTPUT + fi + + - name: Comment result + env: + RESULT: |- + ${{ steps.results.outputs.result == 'pass' && ':white_check_mark: The Bitwise Reproducibility Check Succeeded :white_check_mark:' || ':x: The Bitwise Reproducibility Check Failed :x:' }} + CONFIG_REF_URL: '[${{ needs.prepare-command.outputs.config-short-hash }}](${{ github.server_url}}/${{ github.repository }}/tree/${{ needs.prepare-command.outputs.config-hash }})' + COMPARED_CONFIG_REF_URL: '[${{ needs.prepare-command.outputs.compared-config-short-hash }}](${{ github.server_url }}/${{ github.repository }}/tree/${{ needs.prepare-command.outputs.compared-config-hash }})' + uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: | + ${{ env.RESULT }} + + When comparing: + + - `${{ needs.prepare-command.outputs.config-ref }}` (checksums created using commit ${{ env.CONFIG_REF_URL }}), against + - `${{ needs.prepare-command.outputs.compared-config-ref }}` (checksums in commit ${{ env.COMPARED_CONFIG_REF_URL }}) + + ${{ (needs.prepare-command.outputs.commit-requested == 'true' && steps.results.outputs.result == 'fail') && ':wrench: The checksums will be committed to this PR, as they differ.' || '' }} + +
+ Further information + + The experiment can be found on Gadi at `${{ needs.repro.outputs.experiment-location }}`, and the test results at ${{ steps.results.outputs.check-run-url }}. + + The checksums generated by this `!test` command are found in the `testing/checksum` directory of ${{ needs.repro.outputs.artifact-url }}. + + The checksums compared against are found here ${{ github.server_url }}/${{ github.repository }}/tree/${{ needs.prepare-command.outputs.compared-config-hash }}/testing/checksum + +
+ + commit: + name: Commit Result + if: needs.prepare-command.outputs.commit-requested == 'true' && needs.check-repro.outputs.result == 'fail' + needs: + - prepare-command + - repro + - check-repro + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + env: + ARTIFACT_LOCAL_LOCATION: /opt/artifact + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }} + + - name: Checkout Associated PR ${{ github.event.issue.number }} + # Since the trigger for this workflow was on.issue_comment, we need + # to do a bit more wrangling to checkout the pull request + run: gh pr checkout ${{ github.event.issue.number }} + + - name: Download Newly Created Checksum + uses: actions/download-artifact@v4 + with: + name: ${{ needs.repro.outputs.artifact-name }} + path: ${{ env.ARTIFACT_LOCAL_LOCATION }} + + - name: Update files + # This will copy checksums from the artifact to the repo + run: | + mkdir testing + cp --recursive --verbose ${{ env.ARTIFACT_LOCAL_LOCATION }}/*/* testing + + - name: Import Commit-Signing Key + uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6.1.0 + with: + gpg_private_key: ${{ secrets.GH_ACTIONS_BOT_GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GH_ACTIONS_BOT_GPG_PASSPHRASE }} + git_config_global: true + git_committer_name: ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} + git_committer_email: ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} + git_user_signingkey: true + git_commit_gpgsign: true + git_tag_gpgsign: true + + - name: Commit and Push Updates + run: | + git add . + git commit -m "Updated checksums as part of ${{ env.RUN_URL }}" + git push + + failure-notifier: + name: Notify PR of Workflow Failure + # We need the last jobs as 'needs' on the failure notifier so + # any of the dependent jobs that fail are covered here + needs: + - permission-check + - prepare-command + - check-repro + - commit + if: failure() + runs-on: ubuntu-latest + steps: + - uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: >- + :x: `!test` Command Failed :x: + ${{ needs.prepare-command.result == 'failure' && format('The command given could not be parsed correctly. Usage: {0}', env.USAGE) || '' }} + ${{ needs.permission-check.result == 'failure' && 'You do not have at least write permissions on this repository.' || '' }} + ${{ needs.commit.result == 'failure' && 'There was a problem committing the result of the reproducibility run.' || '' }} + See ${{ env.RUN_URL }} diff --git a/README.md b/README.md index f2a9827..be4f191 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,7 @@ This repository houses pytests that are used as part CI checks for model configu The checksum pytests are used for reproducibility CI checks in this repository. The quick configuration tests are used in any repository that calls `config-pr-1-ci.yml` or is templated by [`ACCESS-NRI/model-configs-template](https://github.com/ACCESS-NRI/model-configs-template). For example, [ACCESS-NRI/access-om2-configs](https://github.com/ACCESS-NRI/access-om2-configs). -Code from these pytests is adapted from COSIMAS's ACCESS-OM2's [ -bit reproducibility tests](https://github.com/COSIMA/access-om2/blob/master/test/test_bit_reproducibility.py). +Code from these pytests is adapted from COSIMAS's ACCESS-OM2's [bit reproducibility tests](https://github.com/COSIMA/access-om2/blob/master/test/test_bit_reproducibility.py). ## Pytests @@ -110,11 +109,11 @@ The `config-*.yml`, `generate-checksums.yml` and `test-repro.yml` workflows are Currently, these repositories make use of the reusable CI: -* [access-om2-configs](https://github.com/ACCESS-NRI/access-om2-configs) -* [access-esm1.5-configs](https://github.com/ACCESS-NRI/access-esm1.5-configs) -* [access-esm1.6-configs](https://github.com/ACCESS-NRI/access-esm1.6-configs) -* [access-om3-configs](https://github.com/ACCESS-NRI/access-om3-configs) -* [access-om3-wav-configs](https://github.com/ACCESS-NRI/access-om3-wav-configs) +- [access-om2-configs](https://github.com/ACCESS-NRI/access-om2-configs) +- [access-esm1.5-configs](https://github.com/ACCESS-NRI/access-esm1.5-configs) +- [access-esm1.6-configs](https://github.com/ACCESS-NRI/access-esm1.6-configs) +- [access-om3-configs](https://github.com/ACCESS-NRI/access-om3-configs) +- [access-om3-wav-configs](https://github.com/ACCESS-NRI/access-om3-wav-configs) Below is information on the use of these workflows. @@ -148,3 +147,35 @@ Using it has some requirements outside of just filling in the inputs: One must h - `vars.EXPERIMENTS_LOCATION` - directory on the deployment target that will contain all the experiments used during testing of reproducibility across multiple runs of this workflow (ex. `/scratch/some/directory/experiments`) - `vars.MODULE_LOCATION` - directory on the deployment target that contains module files for payu used during reproducibility testing (ex. `/g/data/vk83/modules`) - `vars.PRERELEASE_MODULE_LOCATION` - directory on the deployment target that contains module files for development version of payu (ex. `/g/data/vk83/prerelease/modules`) + +#### `config-comment-test` Reusable Workflow + +This comment command allows a repro check from any branch, rather than just the `dev-*` -> `release-*` PR pipeline. + +It requires all the `secrets`/`vars` defined on the caller (as above), as well as `vars.CONFIG_CI_SCHEMA_VERSION`. + +Usage is as follows: + +```txt +!test TYPE [commit] +``` + +Where `TYPE` is a test suite that we support. Currently, this consists of `repro`. + +The commands are all case sensitive and require lower case. + +Using `commit` as an option will commit the result of the test to the PR, provided the commenter has at least `write` permission, and the checksums differ. + +##### `!test repro` + +`!test repro`: will compare the `HEAD` of the current PR source branch against the common ancestor on the target branch. For example, in the below diagram we would be comparing generated checksums in `C` against checksums already stored at `A`: + +```txt +D (PR Target Branch) +| +| C (PR Source Branch) +| | +| B +|/ +A (Common Ancestor) +```