From 9ba6c00b2a3f3a0c6e5bd025145c704fd68b039d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 7 Jun 2023 09:40:31 -0400 Subject: [PATCH 001/150] Update driver.sh no op change for test --- ci/scripts/driver.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index d688e8fc61..f861acb71b 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -13,6 +13,7 @@ set -eux # PR number and calls clone-build_ci.sh to perform a clone and full build from $(HOMEgfs)/sorc # of the PR. It then is ready to run a suite of regression tests with various # configurations with run_tests.py. +# No-op for test ####################################################################################### ################################################################# From e6fb30bf10d89a6ab4159972f2e3e7c0d996d382 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 7 Jun 2023 09:40:31 -0400 Subject: [PATCH 002/150] Update driver.sh no op change for test --- ci/scripts/driver.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index 425d5fbbfd..c913a4aa96 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -13,6 +13,7 @@ set -eux # PR number and calls clone-build_ci.sh to perform a clone and full build from $(HOMEgfs)/sorc # of the PR. It then is ready to run a suite of regression tests with various # configurations with run_tests.py. +# No-op for test ####################################################################################### ################################################################# From daf3f90ff899e3273f2069414b9fee0e39653096 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 28 Jun 2023 20:34:58 +0000 Subject: [PATCH 003/150] added selfhosted.yaml test file --- .github/workflows/selfhosted.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/selfhosted.yaml diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml new file mode 100644 index 0000000000..41d633ba25 --- /dev/null +++ b/.github/workflows/selfhosted.yaml @@ -0,0 +1,18 @@ +name: test-self-hosted-runner + +on: workflow_dispatch + +defaults: + run: + shell: bash + +jobs: + + test-self-hosted-runner: + + runs-on: amzonlinx_tmcg + timeout-minutes: 600 + + steps: + - name: validate-workflow + run: /home/tmcguinness/temp/hello.sh \ No newline at end of file From 9481e51e4754e7618712c652c590afae28c4434e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 28 Jun 2023 20:52:37 +0000 Subject: [PATCH 004/150] updated for merge --- .github/workflows/selfhosted.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index 41d633ba25..e3dbfc39ea 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,3 +1,4 @@ +# Test Self-hosted-Runner name: test-self-hosted-runner on: workflow_dispatch @@ -15,4 +16,4 @@ jobs: steps: - name: validate-workflow - run: /home/tmcguinness/temp/hello.sh \ No newline at end of file + run: /home/tmcguinness/temp/hello.sh From e018b99e215cc62e143e38bfcbf33c3117b5d0f6 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 28 Jun 2023 21:29:20 +0000 Subject: [PATCH 005/150] changed name of Action --- .github/workflows/selfhosted.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index e3dbfc39ea..6967ee824a 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,7 +1,7 @@ # Test Self-hosted-Runner -name: test-self-hosted-runner +name: SelfRunner -on: workflow_dispatch +on: push defaults: run: From 6b64b585c5f65262598b54634bc0b16a72aadc15 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 29 Jun 2023 10:22:36 -0500 Subject: [PATCH 006/150] Added GitHub action for running build script on Orion --- .github/workflows/selfhosted.yaml | 36 +++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index 6967ee824a..e64a5e18b5 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,7 +1,8 @@ -# Test Self-hosted-Runner -name: SelfRunner +name: OrionRunner -on: push +on: + label: + types: [created, deleted] defaults: run: @@ -9,11 +10,32 @@ defaults: jobs: - test-self-hosted-runner: + getlabels: + runs-on: ubuntu-latest + outputs: + labels: ${{ steps.id.outputs.labels }} + steps: + - name: Get Label Steps + id: id + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO_NAME: ${{ github.event.repository.name }} + PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + run: | + LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')" + LABELS=$(echo "$LABELS1" | tr '\n' ' ') + echo "labels=$LABELS" >> $GITHUB_OUTPUT - runs-on: amzonlinx_tmcg + clone-build: + if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Build') + runs-on: [self-hosted, orion-ready] timeout-minutes: 600 + needs: + - getlabels steps: - - name: validate-workflow - run: /home/tmcguinness/temp/hello.sh + - name: build-clone + env: + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + run: /work2/noaa/global/mterry/global-workflow/ci/scripts/clone-build_ci.sh -p ${{ github.event.pull_request.number }} -d /work2/noaa/stmp/GFS_CI_ROOT/PR/${{ github.event.pull_request.number }} -o /work2/noaa/stmp/GFS_CI_ROOT/PR/output_${{ github.run_id }} From 57c6d7c04ed883a967fc6b2ecce8ad5afd46cfb9 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 29 Jun 2023 10:47:45 -0500 Subject: [PATCH 007/150] no op for testing --- .github/workflows/selfhosted.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index e64a5e18b5..88872c24d7 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,3 +1,4 @@ +# no-op name: OrionRunner on: From 9db0274ae5ca698870ffa02b4915e08104ee491b Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 29 Jun 2023 16:31:35 +0000 Subject: [PATCH 008/150] Update selfhosted.yaml no op test --- .github/workflows/selfhosted.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index 88872c24d7..e64a5e18b5 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,4 +1,3 @@ -# no-op name: OrionRunner on: From ea188e054fac2eb7f95708d9ba406fb5aec3c7c9 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 29 Jun 2023 16:40:50 +0000 Subject: [PATCH 009/150] Update selfhosted.yaml removed second label orion-ready --- .github/workflows/selfhosted.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index e64a5e18b5..ff0a6dc565 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -29,7 +29,7 @@ jobs: clone-build: if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Build') - runs-on: [self-hosted, orion-ready] + runs-on: self-hosted timeout-minutes: 600 needs: - getlabels From bc08d9a7cb78dd20793921c8003f347db84a0a9f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 29 Jun 2023 12:10:40 -0500 Subject: [PATCH 010/150] added second label back --- .github/workflows/selfhosted.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index ff0a6dc565..e64a5e18b5 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -29,7 +29,7 @@ jobs: clone-build: if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Build') - runs-on: self-hosted + runs-on: [self-hosted, orion-ready] timeout-minutes: 600 needs: - getlabels From c0601d58b113ee07db3e6b7b9635e2151b61379d Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 29 Jun 2023 12:34:49 -0500 Subject: [PATCH 011/150] added mkdir in clone script because was in driver --- ci/scripts/clone-build_ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index eb2539753b..427f45fd6b 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -40,6 +40,8 @@ while getopts "p:d:o:h" opt; do esac done +mkdir -p "${repodir}" + cd "${repodir}" || exit 1 # clone copy of repo if [[ -d global-workflow ]]; then From f303727f1832405b3a24de06ba7ac5753b1bc5dc Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 29 Jun 2023 12:38:16 -0500 Subject: [PATCH 012/150] changed path to build script to point to dev space --- .github/workflows/selfhosted.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index e64a5e18b5..06ea7a9a72 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -38,4 +38,4 @@ jobs: - name: build-clone env: REPO_URL: ${{ github.server_url }}/${{ github.repository }} - run: /work2/noaa/global/mterry/global-workflow/ci/scripts/clone-build_ci.sh -p ${{ github.event.pull_request.number }} -d /work2/noaa/stmp/GFS_CI_ROOT/PR/${{ github.event.pull_request.number }} -o /work2/noaa/stmp/GFS_CI_ROOT/PR/output_${{ github.run_id }} + run: /work2/noaa/global/mterry/global-workflow_forked/ci/scripts/clone-build_ci.sh -p ${{ github.event.pull_request.number }} -d /work2/noaa/stmp/GFS_CI_ROOT/PR/${{ github.event.pull_request.number }} -o /work2/noaa/stmp/GFS_CI_ROOT/PR/output_${{ github.run_id }} From 0f62aa4d87f5c45d3d5a465361ed9fd7e8b32c52 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 29 Jun 2023 13:51:15 -0500 Subject: [PATCH 013/150] hard coded GH in clone-build script --- ci/scripts/clone-build_ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index 427f45fd6b..f43714b093 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -40,6 +40,8 @@ while getopts "p:d:o:h" opt; do esac done + +GH=/home/mterry/bin/gh mkdir -p "${repodir}" cd "${repodir}" || exit 1 From 038a9b92984cb7dff91145fe0258aeff46fb79fa Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 29 Jun 2023 19:28:56 +0000 Subject: [PATCH 014/150] Update selfhosted.yaml added pull request TARGET --- .github/workflows/selfhosted.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index 06ea7a9a72..a964b4e0e0 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,6 +1,7 @@ name: OrionRunner on: + pull_request_target: label: types: [created, deleted] From b4fc77ac756379f99981dd239e07b1aa18535180 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 29 Jun 2023 19:44:57 +0000 Subject: [PATCH 015/150] Update selfhosted.yaml --- .github/workflows/selfhosted.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index a964b4e0e0..0090ac920e 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,7 +1,7 @@ name: OrionRunner on: - pull_request_target: + pull_request: label: types: [created, deleted] From 3486d35aaf358c1c642b60323b0e50ae6393037c Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 29 Jun 2023 15:23:39 -0500 Subject: [PATCH 016/150] pull request labeled --- .github/workflows/selfhosted.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index 06ea7a9a72..cf43a67c77 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,8 +1,8 @@ name: OrionRunner on: - label: - types: [created, deleted] + pull_request: + types: [ labeled ] defaults: run: From 0c4d39375ca14cdfc1d7802b379ab358a594e8ab Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 29 Jun 2023 15:39:48 -0500 Subject: [PATCH 017/150] just create label --- .github/workflows/selfhosted.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml index cf43a67c77..5aff7b5b68 100644 --- a/.github/workflows/selfhosted.yaml +++ b/.github/workflows/selfhosted.yaml @@ -1,8 +1,8 @@ name: OrionRunner on: - pull_request: - types: [ labeled ] + label: + types: [create] defaults: run: From 0869e1a3e64bb3921ec0f88e3288e4e5f6cf3146 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 30 Jun 2023 19:56:23 +0000 Subject: [PATCH 018/150] added workflow action for building on orion with env vars --- .github/workflows/globalworkflow-ci.yaml | 32 ++++++++++++++++++ .github/workflows/selfhosted.yaml | 41 ------------------------ 2 files changed, 32 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/globalworkflow-ci.yaml delete mode 100644 .github/workflows/selfhosted.yaml diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml new file mode 100644 index 0000000000..645fddc86a --- /dev/null +++ b/.github/workflows/globalworkflow-ci.yaml @@ -0,0 +1,32 @@ + +name: globalworkflow-ci-orion +# A GitHub Workflow Action that creates a deployment for a PR, clones, builds and runs a test suite +# This workflow is triggered by a label being added to a PR +# The label name determines the environment to deploy to +# The label name must be one of the following: +# - CI-Orion-Ready +# - CI-Orion-Build +# - CI-Orion-Run + +on: + pull_request: + types: + - labeled [created, edited] + +jobs: + clone-build: + if: github.event.label.name == 'CI-Orion-Build' + runs-on: [self-hosted, orion-ready] + timeout-minutes: 600 + + steps: + - name: build-clone + env: + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + HOMEgfs: ${{ vars.HOMEGFS_ORION }} + GFS_CI_ROOT: ${{ vars.GFS_CI_ROOT_ORION }} + PR: ${{ github.event.pull_request.number }} + run: ${{ env.HOMEgfs }}/ci/scripts/clone-build_ci.sh \ + -p ${{ env.PR }} \ + -d ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }} \ + -o ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }}/output_${{ github.run_id }} \ No newline at end of file diff --git a/.github/workflows/selfhosted.yaml b/.github/workflows/selfhosted.yaml deleted file mode 100644 index 5aff7b5b68..0000000000 --- a/.github/workflows/selfhosted.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: OrionRunner - -on: - label: - types: [create] - -defaults: - run: - shell: bash - -jobs: - - getlabels: - runs-on: ubuntu-latest - outputs: - labels: ${{ steps.id.outputs.labels }} - steps: - - name: Get Label Steps - id: id - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OWNER: ${{ github.repository_owner }} - REPO_NAME: ${{ github.event.repository.name }} - PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - run: | - LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')" - LABELS=$(echo "$LABELS1" | tr '\n' ' ') - echo "labels=$LABELS" >> $GITHUB_OUTPUT - - clone-build: - if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Build') - runs-on: [self-hosted, orion-ready] - timeout-minutes: 600 - needs: - - getlabels - - steps: - - name: build-clone - env: - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - run: /work2/noaa/global/mterry/global-workflow_forked/ci/scripts/clone-build_ci.sh -p ${{ github.event.pull_request.number }} -d /work2/noaa/stmp/GFS_CI_ROOT/PR/${{ github.event.pull_request.number }} -o /work2/noaa/stmp/GFS_CI_ROOT/PR/output_${{ github.run_id }} From 5e388b30d9776cf3c4e8aff280596aa068faa2d7 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 30 Jun 2023 20:25:22 +0000 Subject: [PATCH 019/150] added single global action file --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 645fddc86a..9f2cbea137 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -11,7 +11,7 @@ name: globalworkflow-ci-orion on: pull_request: types: - - labeled [created, edited] + - labeled jobs: clone-build: @@ -29,4 +29,4 @@ jobs: run: ${{ env.HOMEgfs }}/ci/scripts/clone-build_ci.sh \ -p ${{ env.PR }} \ -d ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }} \ - -o ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }}/output_${{ github.run_id }} \ No newline at end of file + -o ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }}/output_${{ github.run_id }} From 8533f21e08a404e903243992acf8e2d5dc1df245 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 30 Jun 2023 20:36:16 +0000 Subject: [PATCH 020/150] added no op to globalworkflow --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 9f2cbea137..258645ccb0 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -4,6 +4,7 @@ name: globalworkflow-ci-orion # This workflow is triggered by a label being added to a PR # The label name determines the environment to deploy to # The label name must be one of the following: +# # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run From cdb8cf9e1347aebe6ca00c71d95d49871f6592dd Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 30 Jun 2023 20:45:54 +0000 Subject: [PATCH 021/150] removed line continuation delemters --- .github/workflows/globalworkflow-ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 258645ccb0..f835bc1e78 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -27,7 +27,7 @@ jobs: HOMEgfs: ${{ vars.HOMEGFS_ORION }} GFS_CI_ROOT: ${{ vars.GFS_CI_ROOT_ORION }} PR: ${{ github.event.pull_request.number }} - run: ${{ env.HOMEgfs }}/ci/scripts/clone-build_ci.sh \ - -p ${{ env.PR }} \ - -d ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }} \ + run: ${{ env.HOMEgfs }}/ci/scripts/clone-build_ci.sh + -p ${{ env.PR }} + -d ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }} -o ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }}/output_${{ github.run_id }} From 151facf815ccdb6a1dfea695f28d7490fc8615ad Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 30 Jun 2023 15:50:41 -0500 Subject: [PATCH 022/150] no up testing --- .github/workflows/globalworkflow-ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index f835bc1e78..b876a64673 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -4,7 +4,6 @@ name: globalworkflow-ci-orion # This workflow is triggered by a label being added to a PR # The label name determines the environment to deploy to # The label name must be one of the following: -# # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run From 6937f55ceadab0daf56c78a7f06aca97f63236eb Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 15:42:20 +0000 Subject: [PATCH 023/150] removed the BASH script from the build action --- .github/workflows/globalworkflow-ci.yaml | 37 +++++++++++++++--------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index b876a64673..020b8104f5 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -1,6 +1,5 @@ - -name: globalworkflow-ci-orion -# A GitHub Workflow Action that creates a deployment for a PR, clones, builds and runs a test suite +name: gw-ci-orion +# A GitHub Action that creates a deployment for a PR, clones, builds and runs a test suite # This workflow is triggered by a label being added to a PR # The label name determines the environment to deploy to # The label name must be one of the following: @@ -13,6 +12,9 @@ on: types: - labeled +env: + GFS_CI_ROOT: ${{ vars.GFS_CI_ROOT_ORION }} + jobs: clone-build: if: github.event.label.name == 'CI-Orion-Build' @@ -20,13 +22,22 @@ jobs: timeout-minutes: 600 steps: - - name: build-clone - env: - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - HOMEgfs: ${{ vars.HOMEGFS_ORION }} - GFS_CI_ROOT: ${{ vars.GFS_CI_ROOT_ORION }} - PR: ${{ github.event.pull_request.number }} - run: ${{ env.HOMEgfs }}/ci/scripts/clone-build_ci.sh - -p ${{ env.PR }} - -d ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }} - -o ${{ env.GFS_CI_ROOT }}/PR/${{ env.PR }}/output_${{ github.run_id }} + - name: Clone + uses: actions/checkout@v3 + with: + path: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + + - name: Checkout + run: | + cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + ./checkout.sh # Options e.g. -g -u can be added later + + - name: Build + run: | + cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + ./build_all.sh + + - name: Link + run: | + cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + ./link_workflow.sh From 0d626d9f85d7346b5ca26f3e549275bbf06f3a50 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 15:46:36 +0000 Subject: [PATCH 024/150] no op for testing --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 020b8104f5..669122b18e 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -6,6 +6,7 @@ name: gw-ci-orion # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run +# no op on: pull_request: From a148081efb2c70ea57b80ca9662561a8e0b79ad1 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 16:59:15 +0000 Subject: [PATCH 025/150] added working-directy to actions build --- .github/workflows/globalworkflow-ci.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 669122b18e..44e8ef28e7 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -22,9 +22,13 @@ jobs: runs-on: [self-hosted, orion-ready] timeout-minutes: 600 - steps: + - name: Setup + run: | + mkdir ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + - name: Clone uses: actions/checkout@v3 + working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} with: path: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} From 2b3fca8170678377a2bf37fd1f14d574807bf3dc Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 17:00:09 +0000 Subject: [PATCH 026/150] added working dir to actions build --- .github/workflows/globalworkflow-ci.yaml | 48 ++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/globalworkflow-ci.yaml diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml new file mode 100644 index 0000000000..44e8ef28e7 --- /dev/null +++ b/.github/workflows/globalworkflow-ci.yaml @@ -0,0 +1,48 @@ +name: gw-ci-orion +# A GitHub Action that creates a deployment for a PR, clones, builds and runs a test suite +# This workflow is triggered by a label being added to a PR +# The label name determines the environment to deploy to +# The label name must be one of the following: +# - CI-Orion-Ready +# - CI-Orion-Build +# - CI-Orion-Run +# no op + +on: + pull_request: + types: + - labeled + +env: + GFS_CI_ROOT: ${{ vars.GFS_CI_ROOT_ORION }} + +jobs: + clone-build: + if: github.event.label.name == 'CI-Orion-Build' + runs-on: [self-hosted, orion-ready] + timeout-minutes: 600 + + - name: Setup + run: | + mkdir ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + + - name: Clone + uses: actions/checkout@v3 + working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + with: + path: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + + - name: Checkout + run: | + cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + ./checkout.sh # Options e.g. -g -u can be added later + + - name: Build + run: | + cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + ./build_all.sh + + - name: Link + run: | + cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + ./link_workflow.sh From 32b1ca9699d1337f07c649cec585ae7dd8beb3b1 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 17:04:01 +0000 Subject: [PATCH 027/150] removed no op for testing --- .github/workflows/globalworkflow-ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 44e8ef28e7..25a299fd69 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -6,7 +6,6 @@ name: gw-ci-orion # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run -# no op on: pull_request: From 07bab9d3cf9a8f6fb7a4f055f7f08b4e74dafff9 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 17:49:43 +0000 Subject: [PATCH 028/150] Update globalworkflow-ci.yaml --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 25a299fd69..b8156e9c2b 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -20,7 +20,7 @@ jobs: if: github.event.label.name == 'CI-Orion-Build' runs-on: [self-hosted, orion-ready] timeout-minutes: 600 - + - name: Setup run: | mkdir ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} From 808cac6839bbdbda69800c56a97cc7bcc16a5a05 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 17:59:08 +0000 Subject: [PATCH 029/150] Update globalworkflow-ci.yaml added working-directory as part of the defaults --- .github/workflows/globalworkflow-ci.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index b8156e9c2b..53e53905f8 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -14,6 +14,10 @@ on: env: GFS_CI_ROOT: ${{ vars.GFS_CI_ROOT_ORION }} + +defaults: + run: + working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} jobs: clone-build: @@ -21,13 +25,13 @@ jobs: runs-on: [self-hosted, orion-ready] timeout-minutes: 600 + steps: - name: Setup run: | mkdir ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} - name: Clone uses: actions/checkout@v3 - working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} with: path: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} From 321656ca62f68f91e1efb33653750ca0dee30628 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 18:04:55 +0000 Subject: [PATCH 030/150] testing clone action --- .github/workflows/globalworkflow-ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 4063a2f39f..b10a23b419 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -6,7 +6,6 @@ name: gw-ci-orion # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run -# no op on: pull_request: From 5e6b089c203e35c16c55649ad41b663c0b612c79 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 18:19:51 +0000 Subject: [PATCH 031/150] Update globalworkflow-ci.yaml updated vars in working-directory --- .github/workflows/globalworkflow-ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index b10a23b419..c840df96d3 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -17,7 +17,7 @@ env: defaults: run: - working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} jobs: clone-build: @@ -33,7 +33,6 @@ jobs: - name: Clone uses: actions/checkout@v3 - working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} with: path: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} From 91c0984f68dea814add1f9322ecfab9b32b84286 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 18:22:06 +0000 Subject: [PATCH 032/150] updated env to vars for workflow dir --- .github/workflows/globalworkflow-ci.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 44e8ef28e7..b10a23b419 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -6,7 +6,6 @@ name: gw-ci-orion # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run -# no op on: pull_request: @@ -15,12 +14,18 @@ on: env: GFS_CI_ROOT: ${{ vars.GFS_CI_ROOT_ORION }} + +defaults: + run: + working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} jobs: clone-build: if: github.event.label.name == 'CI-Orion-Build' runs-on: [self-hosted, orion-ready] timeout-minutes: 600 + + steps: - name: Setup run: | From cae25f0b4ece73ec8efdb083f707a3d10db51950 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 18:23:32 +0000 Subject: [PATCH 033/150] no up test vars --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index b10a23b419..2f1c3443c3 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -6,6 +6,7 @@ name: gw-ci-orion # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run +# NO OP on: pull_request: From c917e94c53529fef91eede253f6d86abba8c2f91 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 19:38:47 +0000 Subject: [PATCH 034/150] hard coded GFS_CI_ROOT on Orion --- .github/workflows/globalworkflow-ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index ec7a23e340..c941f37d19 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -18,7 +18,8 @@ env: defaults: run: - working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} + #working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} + working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/${{ github.event.pull_request.number }} jobs: clone-build: From abebbb7d2e67a70bb163bd9bfc01f4d21967ea10 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 19:39:29 +0000 Subject: [PATCH 035/150] hard coded GFS_CI_ROOT on Orion --- .github/workflows/globalworkflow-ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index b10a23b419..c941f37d19 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -6,6 +6,7 @@ name: gw-ci-orion # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run +# NO OP on: pull_request: @@ -17,7 +18,8 @@ env: defaults: run: - working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + #working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} + working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/${{ github.event.pull_request.number }} jobs: clone-build: @@ -33,7 +35,6 @@ jobs: - name: Clone uses: actions/checkout@v3 - working-directory: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} with: path: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} From 3e633930a5eb3dc61dd1e6a282b57a5e1bb612ed Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 19:41:01 +0000 Subject: [PATCH 036/150] testing working-directory --- .github/workflows/globalworkflow-ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index c941f37d19..c202e31111 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -6,7 +6,6 @@ name: gw-ci-orion # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run -# NO OP on: pull_request: From 6ab9cdac42ba556a6b1ce820a750497d4536baee Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 19:44:32 +0000 Subject: [PATCH 037/150] Update globalworkflow-ci.yaml 155 --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index c202e31111..ec53c9e9cc 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -18,7 +18,7 @@ env: defaults: run: #working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} - working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/${{ github.event.pull_request.number }} + working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/155 jobs: clone-build: From 4f869895eb45ba02d66d87a5e268b01881b6164e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 19:46:05 +0000 Subject: [PATCH 038/150] Update globalworkflow-ci.yaml 155 --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index c941f37d19..30cbede51a 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -19,7 +19,7 @@ env: defaults: run: #working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} - working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/${{ github.event.pull_request.number }} + working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/155 jobs: clone-build: From 1663eb8145a45405fb44ae34332c9c93787a69ec Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 20:08:00 +0000 Subject: [PATCH 039/150] Update globalworkflow-ci.yaml --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 30cbede51a..edde9eb64a 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -19,7 +19,7 @@ env: defaults: run: #working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} - working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/155 + working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/156 jobs: clone-build: From f069674ab0a4e3b41e8d70f16c8e423f444ef9a9 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 20:10:42 +0000 Subject: [PATCH 040/150] Update globalworkflow-ci.yaml --- .github/workflows/globalworkflow-ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index ec53c9e9cc..fff22346f5 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -6,6 +6,7 @@ name: gw-ci-orion # - CI-Orion-Ready # - CI-Orion-Build # - CI-Orion-Run +# no op on: pull_request: @@ -18,7 +19,7 @@ env: defaults: run: #working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} - working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/155 + working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/156 jobs: clone-build: From 7cb1a7635c31022c247e54b3628037bead54bdfc Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 20:39:10 +0000 Subject: [PATCH 041/150] Update globalworkflow-ci.yaml expermenting with workdir relative path --- .github/workflows/globalworkflow-ci.yaml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 855be7dec9..f0dc7e50a0 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -12,14 +12,6 @@ on: types: - labeled -env: - GFS_CI_ROOT: ${{ vars.GFS_CI_ROOT_ORION }} - -defaults: - run: - #working-directory: ${{ vars.GFS_CI_ROOT_ORION }}/PR/${{ github.event.pull_request.number }} - working-directory: /work2/noaa/stmp/GFS_CI_ROOT/PR/156 - jobs: clone-build: if: github.event.label.name == 'CI-Orion-Build' @@ -30,24 +22,24 @@ jobs: - name: Setup run: | - mkdir ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + mkdir {{ github.event.pull_request.number }} - name: Clone uses: actions/checkout@v3 with: - path: ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }} + path: ${{ github.event.pull_request.number }} - name: Checkout run: | - cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + cd ${{ github.event.pull_request.number }}/sorc ./checkout.sh # Options e.g. -g -u can be added later - name: Build run: | - cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + cd ${{ github.event.pull_request.number }}/sorc ./build_all.sh - name: Link run: | - cd ${{ env.GFS_CI_ROOT }}/PR/${{ github.event.pull_request.number }}/sorc + cd ${{ github.event.pull_request.number }}/sorc ./link_workflow.sh From 207749a39a40df419dc60014ecf713964e521190 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 20:48:11 +0000 Subject: [PATCH 042/150] added relative path for PR on clone build --- .github/workflows/globalworkflow-ci.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index f0dc7e50a0..6dc9181fd1 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -20,10 +20,6 @@ jobs: steps: - - name: Setup - run: | - mkdir {{ github.event.pull_request.number }} - - name: Clone uses: actions/checkout@v3 with: From 1d20d2401ea1acf7b5333719be45a7accb837847 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 21:36:14 +0000 Subject: [PATCH 043/150] added first pass of createing experment --- .github/workflows/globalworkflow-ci.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 6dc9181fd1..d892ab4238 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -39,3 +39,12 @@ jobs: run: | cd ${{ github.event.pull_request.number }}/sorc ./link_workflow.sh + + - name: create-experiments + run: | + cd ${{ github.event.pull_request.number }}/workflow + source gw_setup.sh + source ${{ github.event.pull_request.number }}/ci/platforms/orion.sh + cd ${{ github.event.pull_request.number }}/ci/scripts + ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} + From 3b1b7c6058809fc989780e64fb9467e65ee0a673 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 22:18:03 +0000 Subject: [PATCH 044/150] added label updates and paths in creating experment --- .github/workflows/globalworkflow-ci.yaml | 25 ++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index d892ab4238..66739e7257 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -14,7 +14,7 @@ on: jobs: clone-build: - if: github.event.label.name == 'CI-Orion-Build' + if: github.event.label.name == 'CI-Orion-Ready' runs-on: [self-hosted, orion-ready] timeout-minutes: 600 @@ -30,6 +30,24 @@ jobs: cd ${{ github.event.pull_request.number }}/sorc ./checkout.sh # Options e.g. -g -u can be added later + - name: update label + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: 'CI-Orion-Ready' + }) + github.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['CI-Orion-Building'] + }) + - name: Build run: | cd ${{ github.event.pull_request.number }}/sorc @@ -44,7 +62,10 @@ jobs: run: | cd ${{ github.event.pull_request.number }}/workflow source gw_setup.sh - source ${{ github.event.pull_request.number }}/ci/platforms/orion.sh + cd ${{ github.event.pull_request.number }}/ci/platforms + source orion.sh cd ${{ github.event.pull_request.number }}/ci/scripts ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} + + From 65af29126b95f212ef61a447f36eff8158faba75 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 22:26:44 +0000 Subject: [PATCH 045/150] added dependances between jobs --- .github/workflows/globalworkflow-ci.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 66739e7257..35a41dc8ae 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -31,6 +31,7 @@ jobs: ./checkout.sh # Options e.g. -g -u can be added later - name: update label + needs: Checkout uses: actions/github-script@v3 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -49,23 +50,23 @@ jobs: }) - name: Build + needs: Checkout run: | cd ${{ github.event.pull_request.number }}/sorc ./build_all.sh - name: Link + needs: Build run: | cd ${{ github.event.pull_request.number }}/sorc ./link_workflow.sh - name: create-experiments + needs: Link run: | cd ${{ github.event.pull_request.number }}/workflow source gw_setup.sh cd ${{ github.event.pull_request.number }}/ci/platforms source orion.sh cd ${{ github.event.pull_request.number }}/ci/scripts - ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} - - - + ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} \ No newline at end of file From 5d95a6408f5997e5a5e2254f1a416df6a152c474 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 22:31:55 +0000 Subject: [PATCH 046/150] added dependances between build and create experment jobs --- .github/workflows/globalworkflow-ci.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 35a41dc8ae..c2181b8963 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -31,7 +31,6 @@ jobs: ./checkout.sh # Options e.g. -g -u can be added later - name: update label - needs: Checkout uses: actions/github-script@v3 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -50,19 +49,17 @@ jobs: }) - name: Build - needs: Checkout run: | cd ${{ github.event.pull_request.number }}/sorc ./build_all.sh - name: Link - needs: Build run: | cd ${{ github.event.pull_request.number }}/sorc ./link_workflow.sh - - name: create-experiments - needs: Link + create-experiments + needs: clone-build run: | cd ${{ github.event.pull_request.number }}/workflow source gw_setup.sh From abc8a3fe62452bd66378cd7f63ae46ddc1547582 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 3 Jul 2023 22:39:32 +0000 Subject: [PATCH 047/150] Update globalworkflow-ci.yaml last step for creating an experiment updated --- .github/workflows/globalworkflow-ci.yaml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index c2181b8963..aa8fce3fa6 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -58,12 +58,15 @@ jobs: cd ${{ github.event.pull_request.number }}/sorc ./link_workflow.sh - create-experiments + create-experiments: needs: clone-build - run: | - cd ${{ github.event.pull_request.number }}/workflow - source gw_setup.sh - cd ${{ github.event.pull_request.number }}/ci/platforms - source orion.sh - cd ${{ github.event.pull_request.number }}/ci/scripts - ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} \ No newline at end of file + runs-on: [self-hosted, orion-ready] + steps: + - name: Create Experiments + run: | + cd ${{ github.event.pull_request.number }}/workflow + source gw_setup.sh + cd ${{ github.event.pull_request.number }}/ci/platforms + source orion.sh + cd ${{ github.event.pull_request.number }}/ci/scripts + ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} From 8b1a93f92180134307b4bc4430190d77a3705820 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 00:32:32 +0000 Subject: [PATCH 048/150] Update globalworkflow-ci.yaml updated issue_number: to prNumber --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index aa8fce3fa6..1bc2706425 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -36,13 +36,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | github.issues.removeLabel({ - issue_number: context.issue.number, + issue_number: prNumber, owner: context.repo.owner, repo: context.repo.repo, name: 'CI-Orion-Ready' }) github.issues.addLabels({ - issue_number: context.issue.number, + issue_number: prNumber, owner: context.repo.owner, repo: context.repo.repo, labels: ['CI-Orion-Building'] From dddab51fbc02338c4205b4e165e38388e4bceb5a Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 00:40:01 +0000 Subject: [PATCH 049/150] Update globalworkflow-ci.yaml added context.payload.pull_request.number --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 1bc2706425..2c68c31a9c 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -36,13 +36,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | github.issues.removeLabel({ - issue_number: prNumber, + issue_number: context.payload.pull_request.number, owner: context.repo.owner, repo: context.repo.repo, name: 'CI-Orion-Ready' }) github.issues.addLabels({ - issue_number: prNumber, + issue_number: context.payload.pull_request.number, owner: context.repo.owner, repo: context.repo.repo, labels: ['CI-Orion-Building'] From f173224a3794fec967b5a0278cb6771f9d5bc663 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 00:54:00 +0000 Subject: [PATCH 050/150] Update globalworkflow-ci.yaml updated label update --- .github/workflows/globalworkflow-ci.yaml | 31 +++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 2c68c31a9c..9cdf62938a 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -30,24 +30,27 @@ jobs: cd ${{ github.event.pull_request.number }}/sorc ./checkout.sh # Options e.g. -g -u can be added later - - name: update label - uses: actions/github-script@v3 + - name: Remove label from pull request + uses: actions/github-script@v3.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - github.issues.removeLabel({ - issue_number: context.payload.pull_request.number, - owner: context.repo.owner, - repo: context.repo.repo, + const { context, github } = require('@actions/github'); + const { owner, repo } = context.repo; + await github.issues.removeLabel({ + owner: owner, + repo: repo, + issue_number: context.payload.pull_request.number; name: 'CI-Orion-Ready' - }) - github.issues.addLabels({ - issue_number: context.payload.pull_request.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['CI-Orion-Building'] - }) - + }); + await github.issues.updateLabel({ + owner: owner, + repo: repo, + issue_number: context.payload.pull_request.number; + name: 'CI-Orion-Building' + }); + + - name: Build run: | cd ${{ github.event.pull_request.number }}/sorc From dde0927021087452be1a3ed41fee3a34e41bb26c Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 01:05:53 +0000 Subject: [PATCH 051/150] Update globalworkflow-ci.yaml more --- .github/workflows/globalworkflow-ci.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 9cdf62938a..655f63e6b5 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -28,7 +28,7 @@ jobs: - name: Checkout run: | cd ${{ github.event.pull_request.number }}/sorc - ./checkout.sh # Options e.g. -g -u can be added later + # ./checkout.sh # Options e.g. -g -u can be added later - name: Remove label from pull request uses: actions/github-script@v3.1.0 @@ -37,16 +37,17 @@ jobs: script: | const { context, github } = require('@actions/github'); const { owner, repo } = context.repo; + const prNumber = context.payload.pull_request.number; await github.issues.removeLabel({ owner: owner, repo: repo, - issue_number: context.payload.pull_request.number; + issue_number: prNumber; name: 'CI-Orion-Ready' }); await github.issues.updateLabel({ owner: owner, repo: repo, - issue_number: context.payload.pull_request.number; + issue_number: prNumber; name: 'CI-Orion-Building' }); From bc0515825767371fb70ee80f824502113ead8991 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 01:28:46 +0000 Subject: [PATCH 052/150] Update globalworkflow-ci.yaml skipping label updates for now --- .github/workflows/globalworkflow-ci.yaml | 32 +++++------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 655f63e6b5..9f3d3eb2b2 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -28,40 +28,18 @@ jobs: - name: Checkout run: | cd ${{ github.event.pull_request.number }}/sorc - # ./checkout.sh # Options e.g. -g -u can be added later - - - name: Remove label from pull request - uses: actions/github-script@v3.1.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { context, github } = require('@actions/github'); - const { owner, repo } = context.repo; - const prNumber = context.payload.pull_request.number; - await github.issues.removeLabel({ - owner: owner, - repo: repo, - issue_number: prNumber; - name: 'CI-Orion-Ready' - }); - await github.issues.updateLabel({ - owner: owner, - repo: repo, - issue_number: prNumber; - name: 'CI-Orion-Building' - }); - + ./checkout.sh # Options e.g. -g -u can be added later - name: Build run: | cd ${{ github.event.pull_request.number }}/sorc - ./build_all.sh + #./build_all.sh - name: Link run: | cd ${{ github.event.pull_request.number }}/sorc - ./link_workflow.sh - + #./link_workflow.sh + create-experiments: needs: clone-build runs-on: [self-hosted, orion-ready] @@ -74,3 +52,5 @@ jobs: source orion.sh cd ${{ github.event.pull_request.number }}/ci/scripts ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} + + From bf53fdd7bd68d207392d7fca3adb35777885ce77 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 01:40:10 +0000 Subject: [PATCH 053/150] Update globalworkflow-ci.yaml paths to repo for setup scripts --- .github/workflows/globalworkflow-ci.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 9f3d3eb2b2..d9fbb73ae8 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -45,12 +45,15 @@ jobs: runs-on: [self-hosted, orion-ready] steps: - name: Create Experiments + env: + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + run: | - cd ${{ github.event.pull_request.number }}/workflow + cd $HOMEgfs/workflow source gw_setup.sh - cd ${{ github.event.pull_request.number }}/ci/platforms + cd $HOMEgfs/ci/platforms source orion.sh - cd ${{ github.event.pull_request.number }}/ci/scripts + cd $Homegfs/ci/scripts ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} From 6507fcbee55f0fd9f2a89f707325376224db3ef7 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 01:44:18 +0000 Subject: [PATCH 054/150] Update globalworkflow-ci.yaml HOMEgfs not homegfs --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index d9fbb73ae8..2e9c16ac3b 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -53,7 +53,7 @@ jobs: source gw_setup.sh cd $HOMEgfs/ci/platforms source orion.sh - cd $Homegfs/ci/scripts + cd $HOMEgfs/ci/scripts ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} From 599646d67dea16858dd47223578eab439e5bd61b Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 01:49:42 +0000 Subject: [PATCH 055/150] Update globalworkflow-ci.yaml forgot second dash on dir switch --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 2e9c16ac3b..7a3d438255 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -54,6 +54,6 @@ jobs: cd $HOMEgfs/ci/platforms source orion.sh cd $HOMEgfs/ci/scripts - ./create_experiment.py --yaml ../cases/C48_S2S.yaml -dir ${{ github.event.pull_request.head.ref }} + ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir ${{ github.event.pull_request.head.ref }} From 3367396b66de2398df8a2ee4f9ae872679f0211a Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 02:06:24 +0000 Subject: [PATCH 056/150] Update globalworkflow-ci.yaml more details in paths to HOMEgfs --- .github/workflows/globalworkflow-ci.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 7a3d438255..49f2b5290e 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -46,14 +46,15 @@ jobs: steps: - name: Create Experiments env: - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} + HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} run: | - cd $HOMEgfs/workflow + cd $HOMEGFS/workflow source gw_setup.sh - cd $HOMEgfs/ci/platforms + cd $HOMEGFS/ci/platforms source orion.sh - cd $HOMEgfs/ci/scripts + cd $HOMEGFS/ci/scripts ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir ${{ github.event.pull_request.head.ref }} From f6ed6eaef686521c9b0c1419cfa5c720de1ddd88 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 02:19:17 +0000 Subject: [PATCH 057/150] Update globalworkflow-ci.yaml --dir should be PR HOMEGFS too --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 49f2b5290e..a4e05343a0 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -55,6 +55,6 @@ jobs: cd $HOMEGFS/ci/platforms source orion.sh cd $HOMEGFS/ci/scripts - ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir ${{ github.event.pull_request.head.ref }} + ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir $HOMEGFS From a53e9bbb718e3c4c4e1e7a3b03a562a73060d863 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 02:34:37 +0000 Subject: [PATCH 058/150] Update globalworkflow-ci.yaml Neeced Runtests --- .github/workflows/globalworkflow-ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index a4e05343a0..820e2eb2b5 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -47,9 +47,11 @@ jobs: - name: Create Experiments env: HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} - HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} + HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} + RUNTESTS: ${{ github.workspace }}/RUNTESTS run: | + mkdir -p $RUNTESTS cd $HOMEGFS/workflow source gw_setup.sh cd $HOMEGFS/ci/platforms From 8f653a1527401f7b8b104acb79e9b5dc9eed4408 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 02:42:10 +0000 Subject: [PATCH 059/150] Update globalworkflow-ci.yaml hard coded pslot --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 820e2eb2b5..56932cb3dc 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -49,6 +49,7 @@ jobs: HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} RUNTESTS: ${{ github.workspace }}/RUNTESTS + pslot: C48_S2S run: | mkdir -p $RUNTESTS From bde338967231f5638252ac7a6949f7921e54dd15 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 4 Jul 2023 03:36:32 +0000 Subject: [PATCH 060/150] Update globalworkflow-ci.yaml removed debug comments so really builds now --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 56932cb3dc..07d582f987 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -33,12 +33,12 @@ jobs: - name: Build run: | cd ${{ github.event.pull_request.number }}/sorc - #./build_all.sh + ./build_all.sh - name: Link run: | cd ${{ github.event.pull_request.number }}/sorc - #./link_workflow.sh + ./link_workflow.sh create-experiments: needs: clone-build From 229dd0983087d167476f25ec4916a6125d7daaf3 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 13:38:58 +0000 Subject: [PATCH 061/150] Delete globalworkflow-ci.yaml delete workflows --- .github/workflows/globalworkflow-ci.yaml | 63 ------------------------ 1 file changed, 63 deletions(-) delete mode 100644 .github/workflows/globalworkflow-ci.yaml diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml deleted file mode 100644 index 07d582f987..0000000000 --- a/.github/workflows/globalworkflow-ci.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: gw-ci-orion -# A GitHub Action that creates a deployment for a PR, clones, builds and runs a test suite -# This workflow is triggered by a label being added to a PR -# The label name determines the environment to deploy to -# The label name must be one of the following: -# - CI-Orion-Ready -# - CI-Orion-Build -# - CI-Orion-Run - -on: - pull_request: - types: - - labeled - -jobs: - clone-build: - if: github.event.label.name == 'CI-Orion-Ready' - runs-on: [self-hosted, orion-ready] - timeout-minutes: 600 - - steps: - - - name: Clone - uses: actions/checkout@v3 - with: - path: ${{ github.event.pull_request.number }} - - - name: Checkout - run: | - cd ${{ github.event.pull_request.number }}/sorc - ./checkout.sh # Options e.g. -g -u can be added later - - - name: Build - run: | - cd ${{ github.event.pull_request.number }}/sorc - ./build_all.sh - - - name: Link - run: | - cd ${{ github.event.pull_request.number }}/sorc - ./link_workflow.sh - - create-experiments: - needs: clone-build - runs-on: [self-hosted, orion-ready] - steps: - - name: Create Experiments - env: - HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} - HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} - RUNTESTS: ${{ github.workspace }}/RUNTESTS - pslot: C48_S2S - - run: | - mkdir -p $RUNTESTS - cd $HOMEGFS/workflow - source gw_setup.sh - cd $HOMEGFS/ci/platforms - source orion.sh - cd $HOMEGFS/ci/scripts - ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir $HOMEGFS - - From 845181d3e53eb44b30e664511b836ccd85f7d1bd Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 5 Jul 2023 08:45:22 -0500 Subject: [PATCH 062/150] no op --- ci/scripts/check_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index 40cd3a7c3e..067e4af585 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -6,7 +6,7 @@ set -eux # running rocotostat on each to determine if the experiment has # succeeded or faild. This script is intended # to run from within a cron job in the CI Managers account -# Abstract TODO +# Abstract no op ##################################################################################### HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" From c9e09007bee6a42df74a48239d99c87f22c5bf9d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 13:55:02 +0000 Subject: [PATCH 063/150] adding back global workflow actions --- .github/workflows/globalworkflow-ci.yaml | 63 ++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/globalworkflow-ci.yaml diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml new file mode 100644 index 0000000000..07d582f987 --- /dev/null +++ b/.github/workflows/globalworkflow-ci.yaml @@ -0,0 +1,63 @@ +name: gw-ci-orion +# A GitHub Action that creates a deployment for a PR, clones, builds and runs a test suite +# This workflow is triggered by a label being added to a PR +# The label name determines the environment to deploy to +# The label name must be one of the following: +# - CI-Orion-Ready +# - CI-Orion-Build +# - CI-Orion-Run + +on: + pull_request: + types: + - labeled + +jobs: + clone-build: + if: github.event.label.name == 'CI-Orion-Ready' + runs-on: [self-hosted, orion-ready] + timeout-minutes: 600 + + steps: + + - name: Clone + uses: actions/checkout@v3 + with: + path: ${{ github.event.pull_request.number }} + + - name: Checkout + run: | + cd ${{ github.event.pull_request.number }}/sorc + ./checkout.sh # Options e.g. -g -u can be added later + + - name: Build + run: | + cd ${{ github.event.pull_request.number }}/sorc + ./build_all.sh + + - name: Link + run: | + cd ${{ github.event.pull_request.number }}/sorc + ./link_workflow.sh + + create-experiments: + needs: clone-build + runs-on: [self-hosted, orion-ready] + steps: + - name: Create Experiments + env: + HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} + HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} + RUNTESTS: ${{ github.workspace }}/RUNTESTS + pslot: C48_S2S + + run: | + mkdir -p $RUNTESTS + cd $HOMEGFS/workflow + source gw_setup.sh + cd $HOMEGFS/ci/platforms + source orion.sh + cd $HOMEGFS/ci/scripts + ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir $HOMEGFS + + From 2219ac72e2178238f31473f92edece04ed8b7ac0 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 14:49:46 +0000 Subject: [PATCH 064/150] Update globalworkflow-ci.yaml ignore conda init error --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 07d582f987..e9078e1a81 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -55,6 +55,7 @@ jobs: mkdir -p $RUNTESTS cd $HOMEGFS/workflow source gw_setup.sh + continue-on-error: true cd $HOMEGFS/ci/platforms source orion.sh cd $HOMEGFS/ci/scripts From b129d20b9c43cacce63238fb6a9b727a3cfd0031 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 14:52:39 +0000 Subject: [PATCH 065/150] Update globalworkflow-ci.yaml skipping kernel builds --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index e9078e1a81..2f549e3078 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -33,12 +33,12 @@ jobs: - name: Build run: | cd ${{ github.event.pull_request.number }}/sorc - ./build_all.sh + #./build_all.sh - name: Link run: | cd ${{ github.event.pull_request.number }}/sorc - ./link_workflow.sh + #./link_workflow.sh create-experiments: needs: clone-build From ea120c6a7cfb2e0588222b9742779669459a4d84 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 14:56:57 +0000 Subject: [PATCH 066/150] Update globalworkflow-ci.yaml moved continue-error before sourcing gwsetup --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 2f549e3078..6f62f89e85 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -54,8 +54,8 @@ jobs: run: | mkdir -p $RUNTESTS cd $HOMEGFS/workflow - source gw_setup.sh continue-on-error: true + source gw_setup.sh cd $HOMEGFS/ci/platforms source orion.sh cd $HOMEGFS/ci/scripts From ec917a39355351b3d3d5a72d1d011bad31c9ac46 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 15:06:17 +0000 Subject: [PATCH 067/150] Update globalworkflow-ci.yaml continue-on-errror just after run --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 6f62f89e85..258341015e 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -52,9 +52,9 @@ jobs: pslot: C48_S2S run: | + continue-on-error: true mkdir -p $RUNTESTS cd $HOMEGFS/workflow - continue-on-error: true source gw_setup.sh cd $HOMEGFS/ci/platforms source orion.sh From a94c748d19995683075e3d5beadd38614efa1eb4 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 15:21:00 +0000 Subject: [PATCH 068/150] Update globalworkflow-ci.yaml moved continue-on-error again --- .github/workflows/globalworkflow-ci.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 258341015e..324b586463 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -43,15 +43,17 @@ jobs: create-experiments: needs: clone-build runs-on: [self-hosted, orion-ready] - steps: - - name: Create Experiments - env: + # TODO conda init error on sourcing gw_setup.sh + continue-on-error: true + steps: + - name: Create Experiments + env: HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} RUNTESTS: ${{ github.workspace }}/RUNTESTS pslot: C48_S2S - run: | + run: | continue-on-error: true mkdir -p $RUNTESTS cd $HOMEGFS/workflow From 1d6472f30d22781a7392169d16fcc2b1feee8f4e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 15:31:09 +0000 Subject: [PATCH 069/150] Update globalworkflow-ci.yaml removed bad continue on error --- .github/workflows/globalworkflow-ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 324b586463..64be071f10 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -54,7 +54,6 @@ jobs: pslot: C48_S2S run: | - continue-on-error: true mkdir -p $RUNTESTS cd $HOMEGFS/workflow source gw_setup.sh From 11f8c93a2a60a9dbd41ed18dca3a218618ad2354 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 16:17:57 +0000 Subject: [PATCH 070/150] Update globalworkflow-ci.yaml still debugging BASH error PS1 unbound commenting out source gw_setup.sh --- .github/workflows/globalworkflow-ci.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 64be071f10..6256552d5d 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -43,8 +43,6 @@ jobs: create-experiments: needs: clone-build runs-on: [self-hosted, orion-ready] - # TODO conda init error on sourcing gw_setup.sh - continue-on-error: true steps: - name: Create Experiments env: @@ -56,7 +54,7 @@ jobs: run: | mkdir -p $RUNTESTS cd $HOMEGFS/workflow - source gw_setup.sh + # source gw_setup.sh cd $HOMEGFS/ci/platforms source orion.sh cd $HOMEGFS/ci/scripts From 43fb5b89ca53c5f1a6a4779de6a2e78fd7dafc3a Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 17:29:35 +0000 Subject: [PATCH 071/150] Update globalworkflow-ci.yaml updated path from COMROT to be a function of PR --- .github/workflows/globalworkflow-ci.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 6256552d5d..89ef0cf63b 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -48,16 +48,14 @@ jobs: env: HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} - RUNTESTS: ${{ github.workspace }}/RUNTESTS + RUNTESTS: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS pslot: C48_S2S run: | mkdir -p $RUNTESTS cd $HOMEGFS/workflow - # source gw_setup.sh - cd $HOMEGFS/ci/platforms + # source gw_setup.sh TODO errors on conda init + cd $HOMEGFS/ci/pl/atforms source orion.sh cd $HOMEGFS/ci/scripts ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir $HOMEGFS - - From 08aa0edfc3dfc48f1730c1d7cc47d943d73ed9bb Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 17:39:44 +0000 Subject: [PATCH 072/150] Update globalworkflow-ci.yaml type with "/" snuck in --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 89ef0cf63b..e193a9519a 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -55,7 +55,7 @@ jobs: mkdir -p $RUNTESTS cd $HOMEGFS/workflow # source gw_setup.sh TODO errors on conda init - cd $HOMEGFS/ci/pl/atforms + cd $HOMEGFS/ci/platforms source orion.sh cd $HOMEGFS/ci/scripts ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir $HOMEGFS From 04a627c46bda3680b36734c7b357897e751f0614 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 19:07:19 +0000 Subject: [PATCH 073/150] added job for running rocotorun and check in self-hosted runner --- .github/workflows/globalworkflow-ci.yaml | 17 ++++ ci/platforms/orion.sh | 1 + ci/scripts/run_ci_loped.sh | 113 +++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100755 ci/scripts/run_ci_loped.sh diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index e193a9519a..ff8640cb08 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -59,3 +59,20 @@ jobs: source orion.sh cd $HOMEGFS/ci/scripts ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir $HOMEGFS + + run-experiments: + needs: create-experiments + runs-on: [self-hosted, orion-ready] + steps: + - name: Run Experiment + env: + pslot: C48_S2S + EXPDIR: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS/C48_S2S/EXPDIR + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + run: | + cd $HOMEgfs/ci/platforms + which gh + source orion.sh + which gh + cd $HOMEgfs/ci/scripts + ./run_ci_looped.sh \ No newline at end of file diff --git a/ci/platforms/orion.sh b/ci/platforms/orion.sh index cebab72cc6..e6be2385f2 100644 --- a/ci/platforms/orion.sh +++ b/ci/platforms/orion.sh @@ -5,3 +5,4 @@ export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR export SLURM_ACCOUNT=nems export max_concurrent_cases=2 export max_concurrent_pr=2 +export GH=/home/mterry/bin/gh \ No newline at end of file diff --git a/ci/scripts/run_ci_loped.sh b/ci/scripts/run_ci_loped.sh new file mode 100755 index 0000000000..ea43463f82 --- /dev/null +++ b/ci/scripts/run_ci_loped.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -eux + +##################################################################################### +# +# Script description: BASH script for checking for cases in a given PR and +# simply running rocotorun on each. This script is intended +# to run from within a cron job in the CI Managers account +# Abstract TODO +##################################################################################### + +scriptname=$(basename "${BASH_SOURCE[0]}") +echo "Begin ${scriptname} at $(date -u)" || true +export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' + +######################################################################### +# Set up runtime environment varibles for accounts on supproted machines +######################################################################### + +source "${HOMEgfs}/ush/detect_machine.sh" +case ${MACHINE_ID} in + hera | orion) + echo "Running Automated Testing on ${MACHINE_ID}" + source "${HOMEgfs}/ci/platforms/${MACHINE_ID}.sh" + ;; + *) + echo "Unsupported platform. Exiting with error." + exit 1 + ;; +esac +set +x +source "${HOMEgfs}/ush/module-setup.sh" +module use "${HOMEgfs}/modulefiles" +module load "module_gwsetup.${MACHINE_ID}" +module list +set -eux +rocotorun=$(command -v rocotorun) +if [[ -z ${rocotorun} ]]; then + echo "rocotorun not found on system" + exit 1 +else + echo "rocotorun being used from ${rocotorun}" +fi +rocotostat=$(command -v rocotostat) +if [[ -z ${rocotostat+x} ]]; then + echo "rocotostat not found on system" + exit 1 +else + echo "rocotostat being used from ${rocotostat}" +fi +rocotocheck=$(command -v rocotocheck) +if [[ -z ${rocotocheck+x} ]]; then + echo "rocotocheck not found on system" + exit 1 +else + echo "rocotocheck being used from ${rocotocheck}" +fi + +RUN_COMPLETE="FALSE" + +while [ ${RUN_COMPLETE} .ne. "TRUE" ]; do + + xml="${EXPDIR}/${pslot}/${pslot}.xml" + db="${EXPDIR}/${pslot}/${pslot}.db" + if [[ ! -f "${db}" ]]; then + echo "Database file ${db} not found, experment ${pslot} failed" + exit 1 + fi + if [[ ! -f "${xml}" ]]; then + echo "XML file ${xml} not found, experment ${pslot} failed" + exit 1 + fi + echo "Running: ${rocotorun} -v 6 -w ${xml} -d ${db}" + "${rocotorun}" -v 10 -w "${xml}" -d "${db}" + + sleep 180 + + id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id') + rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true + num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true + num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true + num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true + echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" || true + num_failed=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true + + if [[ ${num_failed} -ne 0 ]]; then + { + echo "Experiment ${pslot} Terminated: *FAILED*" + echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true + } >> "${HOMEgfs}/output_${id}" + error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true + "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed" + { + echo "Error logs:" + echo "${error_logs}" + } >> "${HOMEgfs}/output_${id}" + sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}" + "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/output_${id}" + sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "PR\/${pr}\/RUNTESTS" | awk '{print $1}' | xargs scancel || true + fi + + if [[ "${num_done}" -eq "${num_cycles}" ]]; then + { + echo "Experiment ${pslot} completed: *SUCCESS*" + echo "Experiment ${pslot} Completed at $(date)" || true + echo "with ${num_succeeded} successfully completed jobs" || true + } >> "${HOMEgfs}/output_${id}" + sed -i "s/\`\`\`//2g" "${HOMEgfs}/output_${id}" + "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/output_${id}" + RUN_COMPLETE="TRUE" + fi + +done \ No newline at end of file From a136fc9e3bcb92735c9a0e30bcfe5add61e0ac52 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 19:10:14 +0000 Subject: [PATCH 074/150] removed comments on building the kernals --- .github/workflows/globalworkflow-ci.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index ff8640cb08..523c62c91a 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -33,12 +33,12 @@ jobs: - name: Build run: | cd ${{ github.event.pull_request.number }}/sorc - #./build_all.sh + ./build_all.sh - name: Link run: | cd ${{ github.event.pull_request.number }}/sorc - #./link_workflow.sh + ./link_workflow.sh create-experiments: needs: clone-build @@ -67,8 +67,11 @@ jobs: - name: Run Experiment env: pslot: C48_S2S - EXPDIR: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS/C48_S2S/EXPDIR - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + pr: ${{ github.event.pull_request.number }} + EXPDIR: ${{ github.workspace }}/${{ env.pr }}/RUNTESTS/C48_S2S/EXPDIR + HOMEgfs: ${{ github.workspace }}/${{ env.pr }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + pr: run: | cd $HOMEgfs/ci/platforms which gh From dfee105da9ca0ed3c2b3ae4ab5e2717d14b32147 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 19:12:27 +0000 Subject: [PATCH 075/150] Update globalworkflow-ci.yaml remove extra pr line --- .github/workflows/globalworkflow-ci.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 523c62c91a..cedc88a21f 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -70,12 +70,11 @@ jobs: pr: ${{ github.event.pull_request.number }} EXPDIR: ${{ github.workspace }}/${{ env.pr }}/RUNTESTS/C48_S2S/EXPDIR HOMEgfs: ${{ github.workspace }}/${{ env.pr }} - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - pr: + REPO_URL: ${{ github.server_url }}/${{ github.repository } run: | cd $HOMEgfs/ci/platforms which gh source orion.sh which gh cd $HOMEgfs/ci/scripts - ./run_ci_looped.sh \ No newline at end of file + ./run_ci_looped.sh From 1da289498bacedabfc10d1384ff660731927cf8b Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 19:13:41 +0000 Subject: [PATCH 076/150] Update globalworkflow-ci.yaml missed a } --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index cedc88a21f..485ffec0fe 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -70,7 +70,7 @@ jobs: pr: ${{ github.event.pull_request.number }} EXPDIR: ${{ github.workspace }}/${{ env.pr }}/RUNTESTS/C48_S2S/EXPDIR HOMEgfs: ${{ github.workspace }}/${{ env.pr }} - REPO_URL: ${{ github.server_url }}/${{ github.repository } + REPO_URL: ${{ github.server_url }}/${{ github.repository }} run: | cd $HOMEgfs/ci/platforms which gh From 1ada88f122eeefcff52e62f71d9cfeab6c192544 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 19:22:23 +0000 Subject: [PATCH 077/150] added wait before running rocotorun again --- ci/scripts/run_ci_loped.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/scripts/run_ci_loped.sh b/ci/scripts/run_ci_loped.sh index ea43463f82..eaaf909e7e 100755 --- a/ci/scripts/run_ci_loped.sh +++ b/ci/scripts/run_ci_loped.sh @@ -73,7 +73,8 @@ while [ ${RUN_COMPLETE} .ne. "TRUE" ]; do echo "Running: ${rocotorun} -v 6 -w ${xml} -d ${db}" "${rocotorun}" -v 10 -w "${xml}" -d "${db}" - sleep 180 + # Wait before running rocotostat + sleep 60 id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id') rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true @@ -110,4 +111,7 @@ while [ ${RUN_COMPLETE} .ne. "TRUE" ]; do RUN_COMPLETE="TRUE" fi + # Wait before running rocotorun again + sleep 300 + done \ No newline at end of file From 015b12d254ab86d1fd9347d08971ab063711d8d9 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 19:50:43 +0000 Subject: [PATCH 078/150] Update globalworkflow-ci.yaml update env.pr to github values --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 485ffec0fe..57f6325cab 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -68,8 +68,8 @@ jobs: env: pslot: C48_S2S pr: ${{ github.event.pull_request.number }} - EXPDIR: ${{ github.workspace }}/${{ env.pr }}/RUNTESTS/C48_S2S/EXPDIR - HOMEgfs: ${{ github.workspace }}/${{ env.pr }} + EXPDIR: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS/C48_S2S/EXPDIR + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} run: | cd $HOMEgfs/ci/platforms From 23f45bdc532fa1838ccd1fea33410c7800e6c71f Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 21:06:15 +0000 Subject: [PATCH 079/150] typo fix in run_ci_looped file name --- ci/scripts/{run_ci_loped.sh => run_ci_looped.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ci/scripts/{run_ci_loped.sh => run_ci_looped.sh} (100%) diff --git a/ci/scripts/run_ci_loped.sh b/ci/scripts/run_ci_looped.sh similarity index 100% rename from ci/scripts/run_ci_loped.sh rename to ci/scripts/run_ci_looped.sh From e397dde0bbbcb458dfabf3e77327ced9b971984a Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 21:22:11 +0000 Subject: [PATCH 080/150] relized conda was init-ed in runners env so put back gw_set.sh --- .github/workflows/globalworkflow-ci.yaml | 4 +--- ci/scripts/run_ci_looped.sh | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 57f6325cab..77e4823e80 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -54,7 +54,7 @@ jobs: run: | mkdir -p $RUNTESTS cd $HOMEGFS/workflow - # source gw_setup.sh TODO errors on conda init + source ./gw_setup.sh cd $HOMEGFS/ci/platforms source orion.sh cd $HOMEGFS/ci/scripts @@ -73,8 +73,6 @@ jobs: REPO_URL: ${{ github.server_url }}/${{ github.repository }} run: | cd $HOMEgfs/ci/platforms - which gh source orion.sh - which gh cd $HOMEgfs/ci/scripts ./run_ci_looped.sh diff --git a/ci/scripts/run_ci_looped.sh b/ci/scripts/run_ci_looped.sh index eaaf909e7e..a59a5f66f2 100755 --- a/ci/scripts/run_ci_looped.sh +++ b/ci/scripts/run_ci_looped.sh @@ -57,8 +57,7 @@ else fi RUN_COMPLETE="FALSE" - -while [ ${RUN_COMPLETE} .ne. "TRUE" ]; do +while [ ${RUN_COMPLETE} == "TRUE" ]; do xml="${EXPDIR}/${pslot}/${pslot}.xml" db="${EXPDIR}/${pslot}/${pslot}.db" From 097c2fd4a4a64f5f5c38ab30a1525580d16712e8 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 21:23:20 +0000 Subject: [PATCH 081/150] Update orion.sh space --- ci/platforms/orion.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/platforms/orion.sh b/ci/platforms/orion.sh index e6be2385f2..1bc5071d34 100644 --- a/ci/platforms/orion.sh +++ b/ci/platforms/orion.sh @@ -5,4 +5,4 @@ export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR export SLURM_ACCOUNT=nems export max_concurrent_cases=2 export max_concurrent_pr=2 -export GH=/home/mterry/bin/gh \ No newline at end of file +export GH=/home/mterry/bin/gh From bac584115f7f505d9c95f7d2c3df236e53173031 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 21:24:21 +0000 Subject: [PATCH 082/150] Update run_ci_looped.sh linting bash --- ci/scripts/run_ci_looped.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/run_ci_looped.sh b/ci/scripts/run_ci_looped.sh index a59a5f66f2..ef813b2127 100755 --- a/ci/scripts/run_ci_looped.sh +++ b/ci/scripts/run_ci_looped.sh @@ -57,7 +57,7 @@ else fi RUN_COMPLETE="FALSE" -while [ ${RUN_COMPLETE} == "TRUE" ]; do +while [[ ${RUN_COMPLETE} == "TRUE" ]]; do xml="${EXPDIR}/${pslot}/${pslot}.xml" db="${EXPDIR}/${pslot}/${pslot}.db" @@ -113,4 +113,4 @@ while [ ${RUN_COMPLETE} == "TRUE" ]; do # Wait before running rocotorun again sleep 300 -done \ No newline at end of file +done From 16124816cf1f16f18871c9c91b912d7fec337646 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 21:25:25 +0000 Subject: [PATCH 083/150] Update check_ci.sh --- ci/scripts/check_ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index 067e4af585..11ea7f29b5 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -6,7 +6,6 @@ set -eux # running rocotostat on each to determine if the experiment has # succeeded or faild. This script is intended # to run from within a cron job in the CI Managers account -# Abstract no op ##################################################################################### HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" From 7be9cb925b85a6721a8376c25459249060dbc508 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 21:53:21 +0000 Subject: [PATCH 084/150] do while RUN_COMPLETE is FALSE --- ci/scripts/run_ci_looped.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/run_ci_looped.sh b/ci/scripts/run_ci_looped.sh index a59a5f66f2..c1048e8f6a 100755 --- a/ci/scripts/run_ci_looped.sh +++ b/ci/scripts/run_ci_looped.sh @@ -57,7 +57,7 @@ else fi RUN_COMPLETE="FALSE" -while [ ${RUN_COMPLETE} == "TRUE" ]; do +while [ ${RUN_COMPLETE} == "FALSE" ]; do xml="${EXPDIR}/${pslot}/${pslot}.xml" db="${EXPDIR}/${pslot}/${pslot}.db" From d8d9a66312cab878114a9fbbdcee8dc57740b089 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 5 Jul 2023 22:06:40 +0000 Subject: [PATCH 085/150] fixed loop logic over rocotorun --- ci/scripts/run_ci_looped.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/scripts/run_ci_looped.sh b/ci/scripts/run_ci_looped.sh index c1048e8f6a..6f007efb26 100755 --- a/ci/scripts/run_ci_looped.sh +++ b/ci/scripts/run_ci_looped.sh @@ -61,10 +61,7 @@ while [ ${RUN_COMPLETE} == "FALSE" ]; do xml="${EXPDIR}/${pslot}/${pslot}.xml" db="${EXPDIR}/${pslot}/${pslot}.db" - if [[ ! -f "${db}" ]]; then - echo "Database file ${db} not found, experment ${pslot} failed" - exit 1 - fi + if [[ ! -f "${xml}" ]]; then echo "XML file ${xml} not found, experment ${pslot} failed" exit 1 @@ -74,6 +71,10 @@ while [ ${RUN_COMPLETE} == "FALSE" ]; do # Wait before running rocotostat sleep 60 + if [[ ! -f "${db}" ]]; then + echo "Database file ${db} not found, experment ${pslot} failed" + exit 1 + fi id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id') rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true @@ -111,6 +112,6 @@ while [ ${RUN_COMPLETE} == "FALSE" ]; do fi # Wait before running rocotorun again - sleep 300 + sleep 240 done \ No newline at end of file From b9be31935d13ff0b70c1a7b5a5cf5d0c61808981 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 16:02:55 +0000 Subject: [PATCH 086/150] bash lint brackets --- ci/scripts/run_ci_looped.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/run_ci_looped.sh b/ci/scripts/run_ci_looped.sh index 0ccc360be9..464429c802 100755 --- a/ci/scripts/run_ci_looped.sh +++ b/ci/scripts/run_ci_looped.sh @@ -57,7 +57,7 @@ else fi RUN_COMPLETE="FALSE" -while [ ${RUN_COMPLETE} == "FALSE" ]; do +while [[ ${RUN_COMPLETE} == "FALSE" ]]; do xml="${EXPDIR}/${pslot}/${pslot}.xml" db="${EXPDIR}/${pslot}/${pslot}.db" From 242e2f4ac7f16e5f843acce825130c09b64bf3de Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 17:02:48 +0000 Subject: [PATCH 087/150] added label updates and comments to PR messaging --- .github/workflows/globalworkflow-ci.yaml | 28 +++++++++++++++++-- .../{run_ci_looped.sh => run-check_ci.sh} | 14 +++++----- 2 files changed, 33 insertions(+), 9 deletions(-) rename ci/scripts/{run_ci_looped.sh => run-check_ci.sh} (92%) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 77e4823e80..663b35b310 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -20,6 +20,19 @@ jobs: steps: + run: | + cd ${{ github.workspace }} + mkdir -p ${{ github.event.pull_request.number }} + cd ${{ github.event.pull_request.number }} + { + echo "Automated global-workflow Testing Results:" + echo '```' + echo "Machine: Orion" + echo "Start: $(date) on $(hostname)" || true + echo "---------------------------------------------------" + } >> ${{ github.workspace }}/${{ github.event.pull_request.number }}/global-workflow.log + gh pr comment ${{ github.event.pull_request.number }} --body "$(cat global-workflow.log)" + - name: Clone uses: actions/checkout@v3 with: @@ -29,11 +42,22 @@ jobs: run: | cd ${{ github.event.pull_request.number }}/sorc ./checkout.sh # Options e.g. -g -u can be added later - + - name: Build + env: + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + MACHINE_ID: orion run: | cd ${{ github.event.pull_request.number }}/sorc ./build_all.sh + if [ $? -ne 0 ]; then + gh pr comment ${{ github.event.pull_request.number }} --body "Automated global-workflow Testing Results: \n\n Build Failed" + gh pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Failed" + exit 1 + else + gh pr comment ${{ github.event.pull_request.number }} --body "Automated global-workflow Testing Results: \n\n Build Passed" + gh pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Running" + fi - name: Link run: | @@ -75,4 +99,4 @@ jobs: cd $HOMEgfs/ci/platforms source orion.sh cd $HOMEgfs/ci/scripts - ./run_ci_looped.sh + ./run-check_ci.sh \ No newline at end of file diff --git a/ci/scripts/run_ci_looped.sh b/ci/scripts/run-check_ci.sh similarity index 92% rename from ci/scripts/run_ci_looped.sh rename to ci/scripts/run-check_ci.sh index 464429c802..20ad701694 100755 --- a/ci/scripts/run_ci_looped.sh +++ b/ci/scripts/run-check_ci.sh @@ -88,15 +88,15 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do { echo "Experiment ${pslot} Terminated: *FAILED*" echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true - } >> "${HOMEgfs}/output_${id}" + } >> "${HOMEgfs}/global-workflow.log" error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed" { echo "Error logs:" echo "${error_logs}" - } >> "${HOMEgfs}/output_${id}" - sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/output_${id}" + } >> "${HOMEgfs}/global-workflow.log" + sed -i "s/\`\`\`//2g" "${HOMEgfs}/global-workflow.log" + "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/global-workflow.log" sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "PR\/${pr}\/RUNTESTS" | awk '{print $1}' | xargs scancel || true fi @@ -105,9 +105,9 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do echo "Experiment ${pslot} completed: *SUCCESS*" echo "Experiment ${pslot} Completed at $(date)" || true echo "with ${num_succeeded} successfully completed jobs" || true - } >> "${HOMEgfs}/output_${id}" - sed -i "s/\`\`\`//2g" "${HOMEgfs}/output_${id}" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/output_${id}" + } >> "${HOMEgfs}/global-workflow.log" + sed -i "s/\`\`\`//2g" "${HOMEgfs}/global-workflow.log" + "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/global-workflow.log" RUN_COMPLETE="TRUE" fi From 61ba7c693acba7ea2c0aa51e88d2e3b24a3683a3 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 17:13:35 +0000 Subject: [PATCH 088/150] Update globalworkflow-ci.yaml moved comments to PR user just after checkout --- .github/workflows/globalworkflow-ci.yaml | 26 ++++++++++-------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 663b35b310..e8eaa22ce6 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -20,19 +20,6 @@ jobs: steps: - run: | - cd ${{ github.workspace }} - mkdir -p ${{ github.event.pull_request.number }} - cd ${{ github.event.pull_request.number }} - { - echo "Automated global-workflow Testing Results:" - echo '```' - echo "Machine: Orion" - echo "Start: $(date) on $(hostname)" || true - echo "---------------------------------------------------" - } >> ${{ github.workspace }}/${{ github.event.pull_request.number }}/global-workflow.log - gh pr comment ${{ github.event.pull_request.number }} --body "$(cat global-workflow.log)" - - name: Clone uses: actions/checkout@v3 with: @@ -41,7 +28,16 @@ jobs: - name: Checkout run: | cd ${{ github.event.pull_request.number }}/sorc - ./checkout.sh # Options e.g. -g -u can be added later + ./checkout.sh # Options e.g. -g -u can be added late + cd ${{ github.event.pull_request.number }} + { + echo "Automated global-workflow Testing Results:" + echo '```' + echo "Machine: Orion" + echo "Start: $(date) on $(hostname)" || true + echo "---------------------------------------------------" + } >> ${{ github.workspace }}/${{ github.event.pull_request.number }}/global-workflow.log + gh pr comment ${{ github.event.pull_request.number }} --body "$(cat global-workflow.log)" - name: Build env: @@ -99,4 +95,4 @@ jobs: cd $HOMEgfs/ci/platforms source orion.sh cd $HOMEgfs/ci/scripts - ./run-check_ci.sh \ No newline at end of file + ./run-check_ci.sh From 9ea55fd730d265e5f26d9f23788301479d78253b Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 17:27:18 +0000 Subject: [PATCH 089/150] Update run-check_ci.sh removed id line that is no longer used --- ci/scripts/run-check_ci.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 20ad701694..c8842f4930 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -75,8 +75,6 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do echo "Database file ${db} not found, experment ${pslot} failed" exit 1 fi - - id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id') rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true From 4f2281536682eff089979159cbb2a8117b850b31 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 17:34:27 +0000 Subject: [PATCH 090/150] Update globalworkflow-ci.yaml updated HOMEgfs paths and cd into dirs --- .github/workflows/globalworkflow-ci.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index e8eaa22ce6..eab9ceac6c 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -26,25 +26,28 @@ jobs: path: ${{ github.event.pull_request.number }} - name: Checkout + env: + HOMEgfs: ${{ github.workspace }}/{{ github.event.pull_request.number }} run: | - cd ${{ github.event.pull_request.number }}/sorc + cd $HOMEgfs/sorc ./checkout.sh # Options e.g. -g -u can be added late - cd ${{ github.event.pull_request.number }} + cd $HOMEgfs { echo "Automated global-workflow Testing Results:" echo '```' echo "Machine: Orion" echo "Start: $(date) on $(hostname)" || true echo "---------------------------------------------------" - } >> ${{ github.workspace }}/${{ github.event.pull_request.number }}/global-workflow.log + } >> global-workflow.log gh pr comment ${{ github.event.pull_request.number }} --body "$(cat global-workflow.log)" - name: Build env: + HOMEgfs: ${{ github.workspace }}/{{ github.event.pull_request.number }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} MACHINE_ID: orion run: | - cd ${{ github.event.pull_request.number }}/sorc + cd $HOMEgfs/sorc ./build_all.sh if [ $? -ne 0 ]; then gh pr comment ${{ github.event.pull_request.number }} --body "Automated global-workflow Testing Results: \n\n Build Failed" From 22abf49c4bf856a76870dcb0fefaf6658ff16fbb Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 17:38:14 +0000 Subject: [PATCH 091/150] Update globalworkflow-ci.yaml missing $ in gitHub PR --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index eab9ceac6c..d5ff5ccef5 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -27,7 +27,7 @@ jobs: - name: Checkout env: - HOMEgfs: ${{ github.workspace }}/{{ github.event.pull_request.number }} + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} run: | cd $HOMEgfs/sorc ./checkout.sh # Options e.g. -g -u can be added late @@ -43,7 +43,7 @@ jobs: - name: Build env: - HOMEgfs: ${{ github.workspace }}/{{ github.event.pull_request.number }} + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} MACHINE_ID: orion run: | From a25adad9438ee5b25fdae17a3752dc0ff40e36a6 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 17:59:30 +0000 Subject: [PATCH 092/150] added pr switch to gh messaging --- .github/workflows/globalworkflow-ci.yaml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index d5ff5ccef5..53c4f1e965 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -28,6 +28,7 @@ jobs: - name: Checkout env: HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} run: | cd $HOMEgfs/sorc ./checkout.sh # Options e.g. -g -u can be added late @@ -39,23 +40,24 @@ jobs: echo "Start: $(date) on $(hostname)" || true echo "---------------------------------------------------" } >> global-workflow.log - gh pr comment ${{ github.event.pull_request.number }} --body "$(cat global-workflow.log)" + gh pr comment ${{ github.event.pull_request.number }} --repo $REPO_URL --body "$(cat global-workflow.log)" - name: Build env: HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} MACHINE_ID: orion + pr: ${{ github.event.pull_request.number }} run: | - cd $HOMEgfs/sorc + cd ${HOMEgfs}/sorc ./build_all.sh if [ $? -ne 0 ]; then - gh pr comment ${{ github.event.pull_request.number }} --body "Automated global-workflow Testing Results: \n\n Build Failed" - gh pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Failed" + gh pr comment ${pr} --repo ${REPO_URL} --body "Automated global-workflow Testing Results: \n\n Build Failed" + gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Failed" exit 1 else - gh pr comment ${{ github.event.pull_request.number }} --body "Automated global-workflow Testing Results: \n\n Build Passed" - gh pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Running" + gh pr comment ${pr} --repo $REPO_URL --body "Automated global-workflow Testing Results: \n\n Build Passed" + gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Running" fi - name: Link @@ -75,13 +77,13 @@ jobs: pslot: C48_S2S run: | - mkdir -p $RUNTESTS - cd $HOMEGFS/workflow + mkdir -p ${RUNTESTS} + cd ${HOMEGFS}/workflow source ./gw_setup.sh - cd $HOMEGFS/ci/platforms + cd ${HOMEGFS}/ci/platforms source orion.sh - cd $HOMEGFS/ci/scripts - ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir $HOMEGFS + cd ${HOMEGFS}/ci/scripts + ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir ${HOMEGFS} run-experiments: needs: create-experiments From 772cc0b8512f51b85bab5cd8a7faf46249299589 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 18:18:14 +0000 Subject: [PATCH 093/150] added step for Building for label updates --- .github/workflows/globalworkflow-ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 53c4f1e965..4895883f4b 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -49,15 +49,16 @@ jobs: MACHINE_ID: orion pr: ${{ github.event.pull_request.number }} run: | + gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building" cd ${HOMEgfs}/sorc ./build_all.sh if [ $? -ne 0 ]; then gh pr comment ${pr} --repo ${REPO_URL} --body "Automated global-workflow Testing Results: \n\n Build Failed" - gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Failed" + gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed" exit 1 else gh pr comment ${pr} --repo $REPO_URL --body "Automated global-workflow Testing Results: \n\n Build Passed" - gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Running" + gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running" fi - name: Link From 759dfdd7dc0d84c76fb118b9bff9486ec5237628 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 19:32:02 +0000 Subject: [PATCH 094/150] updated messaging details for building status --- .github/workflows/globalworkflow-ci.yaml | 51 ++++++++++++++++++------ 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 4895883f4b..e9da4e26f4 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -30,9 +30,7 @@ jobs: HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} run: | - cd $HOMEgfs/sorc - ./checkout.sh # Options e.g. -g -u can be added late - cd $HOMEgfs + cd ${HOMEgfs} { echo "Automated global-workflow Testing Results:" echo '```' @@ -40,31 +38,60 @@ jobs: echo "Start: $(date) on $(hostname)" || true echo "---------------------------------------------------" } >> global-workflow.log - gh pr comment ${{ github.event.pull_request.number }} --repo $REPO_URL --body "$(cat global-workflow.log)" + cd $HOMEgfs/sorc + ./checkout.sh # Options e.g. -g -u can be added late + if [ $? -ne 0 ]; then + { + echo "Checkout: *FAILED*" + echo "Checkout: Failed at $(date)" || true + echo "Checkout: see output at ${PWD}/log.checkout" + } >> ${HOMEgfs}/global-workflow.log + exit 1 + else + { + echo "Checkout: *SUCCESS*" + echo "Checkout: Completed at $(date)" || true + } >> ${HOMEgfs}/global-workflow.log + fi + gh pr comment ${{ github.event.pull_request.number }} --repo $REPO_URL --body-file ${HOMEgfs}/global-workflow.log - name: Build env: - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} MACHINE_ID: orion pr: ${{ github.event.pull_request.number }} run: | gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building" + cd ${HOMEgfs}/sorc ./build_all.sh + + - name: Link + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + MACHINE_ID: orion + pr: ${{ github.event.pull_request.number }} + run: | + cd ${{ github.event.pull_request.number }}/sorc + ./link_workflow.sh + if [ $? -ne 0 ]; then - gh pr comment ${pr} --repo ${REPO_URL} --body "Automated global-workflow Testing Results: \n\n Build Failed" + { + echo "Build: *FAILED*" + echo "Build: Failed at $(date)" || true + echo "Build: see output at ${PWD}/log.build" + } >> ${HOMEgfs}/global-workflow.log + gh pr comment ${pr} --repo ${REPO_URL} --body-file global-workflow.log gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed" exit 1 else - gh pr comment ${pr} --repo $REPO_URL --body "Automated global-workflow Testing Results: \n\n Build Passed" + { + echo "Build: *SUCCESS*" + echo "Build: Completed at $(date)" || true + } >> ${HOMEgfs}/global-workflow.log + gh pr comment ${pr} --repo $REPO_URL --body-file ${HOMEgfs}/global-workflow.log gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running" fi - - - name: Link - run: | - cd ${{ github.event.pull_request.number }}/sorc - ./link_workflow.sh create-experiments: needs: clone-build From 4745e3ace5d0319562997c47f7aac0125394dc81 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 19:34:36 +0000 Subject: [PATCH 095/150] Update globalworkflow-ci.yaml forgot env: on Link --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index e9da4e26f4..73f19448e8 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -67,6 +67,7 @@ jobs: ./build_all.sh - name: Link + env: HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} MACHINE_ID: orion From c722e7eeb79aceafc173c0f5abb6c208f07a3dbd Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 19:46:59 +0000 Subject: [PATCH 096/150] Update globalworkflow-ci.yaml finer details on output for GitHub --- .github/workflows/globalworkflow-ci.yaml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 73f19448e8..052e4d9f9f 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -40,23 +40,10 @@ jobs: } >> global-workflow.log cd $HOMEgfs/sorc ./checkout.sh # Options e.g. -g -u can be added late - if [ $? -ne 0 ]; then - { - echo "Checkout: *FAILED*" - echo "Checkout: Failed at $(date)" || true - echo "Checkout: see output at ${PWD}/log.checkout" - } >> ${HOMEgfs}/global-workflow.log - exit 1 - else - { - echo "Checkout: *SUCCESS*" - echo "Checkout: Completed at $(date)" || true - } >> ${HOMEgfs}/global-workflow.log - fi - gh pr comment ${{ github.event.pull_request.number }} --repo $REPO_URL --body-file ${HOMEgfs}/global-workflow.log - + - name: Build env: + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} REPO_URL: ${{ github.server_url }}/${{ github.repository }} MACHINE_ID: orion pr: ${{ github.event.pull_request.number }} From 72f496a2a32364106c5a326c41314eaa4aec3822 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 21:17:07 +0000 Subject: [PATCH 097/150] added label update Passed after success on run check in run-check BASH script --- ci/scripts/run-check_ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index c8842f4930..a72db232db 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -106,7 +106,9 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do } >> "${HOMEgfs}/global-workflow.log" sed -i "s/\`\`\`//2g" "${HOMEgfs}/global-workflow.log" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/global-workflow.log" + "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed" RUN_COMPLETE="TRUE" + exit 0 fi # Wait before running rocotorun again From 6cc6e86095ba2860d644c1a454157c5658ce638f Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 6 Jul 2023 17:32:35 -0400 Subject: [PATCH 098/150] Update clone-build_ci.sh --- ci/scripts/clone-build_ci.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index f43714b093..427f45fd6b 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -40,8 +40,6 @@ while getopts "p:d:o:h" opt; do esac done - -GH=/home/mterry/bin/gh mkdir -p "${repodir}" cd "${repodir}" || exit 1 From d78d6c2665a84e1d7cbf82c25de9eeb55e032eae Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 6 Jul 2023 23:11:00 -0400 Subject: [PATCH 099/150] eliminate use of GH cli from actions --- .github/workflows/globalworkflow-ci.yaml | 190 +++++++++++++---------- ci/scripts/create_experiment.py | 9 +- ci/scripts/run-check_ci.sh | 31 ++-- 3 files changed, 130 insertions(+), 100 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 052e4d9f9f..b6ef13b6de 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -12,108 +12,140 @@ on: types: - labeled +env: + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} + RUNTESTS: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS + MACHINE_ID: orion + jobs: - clone-build: + checkout-build-link: if: github.event.label.name == 'CI-Orion-Ready' runs-on: [self-hosted, orion-ready] timeout-minutes: 600 - - steps: - - name: Clone + steps: + - name: Checkout uses: actions/checkout@v3 with: path: ${{ github.event.pull_request.number }} - - name: Checkout + - name: Checkout components env: HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} - REPO_URL: ${{ github.server_url }}/${{ github.repository }} run: | - cd ${HOMEgfs} - { - echo "Automated global-workflow Testing Results:" - echo '```' - echo "Machine: Orion" - echo "Start: $(date) on $(hostname)" || true - echo "---------------------------------------------------" - } >> global-workflow.log - cd $HOMEgfs/sorc + cd ${{ env.HOMEgfs }}/sorc ./checkout.sh # Options e.g. -g -u can be added late - - - name: Build - env: - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - MACHINE_ID: orion - pr: ${{ github.event.pull_request.number }} - run: | - gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building" - cd ${HOMEgfs}/sorc + - name: Build executables + uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + with: + labels: "CI-Orion-Ready" + uses: actions-ecosystem/action-add-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + with: + labels: "CI-Orion-Building" + run: | + cd ${{ env.HOMEgfs }}/sorc ./build_all.sh + if: ${{ failure() }} + uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + with: + labels: "CI-Orion-Building" + uses: actions-ecosystem/action-add-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + with: + labels: "CI-Orion-Failed" - - name: Link - env: - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - MACHINE_ID: orion - pr: ${{ github.event.pull_request.number }} + - name: Link artifacts run: | - cd ${{ github.event.pull_request.number }}/sorc + cd ${{ env.HOMEgfs }}/sorc ./link_workflow.sh + if: success() || failure() + uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + with: + labels: "CI-Orion-Building" + if: failure() + uses: actions-ecosystem/action-add-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + with: + labels: "CI-Orion-Failed" + if: success() + uses: actions-ecosystem/action-add-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + with: + labels: "CI-Orion-Running" - if [ $? -ne 0 ]; then - { - echo "Build: *FAILED*" - echo "Build: Failed at $(date)" || true - echo "Build: see output at ${PWD}/log.build" - } >> ${HOMEgfs}/global-workflow.log - gh pr comment ${pr} --repo ${REPO_URL} --body-file global-workflow.log - gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed" - exit 1 - else - { - echo "Build: *SUCCESS*" - echo "Build: Completed at $(date)" || true - } >> ${HOMEgfs}/global-workflow.log - gh pr comment ${pr} --repo $REPO_URL --body-file ${HOMEgfs}/global-workflow.log - gh pr edit --repo ${REPO_URL} ${pr} --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running" - fi - create-experiments: - needs: clone-build + needs: checkout-build-link runs-on: [self-hosted, orion-ready] - steps: - - name: Create Experiments - env: - HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} - HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} - RUNTESTS: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS - pslot: C48_S2S - - run: | - mkdir -p ${RUNTESTS} - cd ${HOMEGFS}/workflow - source ./gw_setup.sh - cd ${HOMEGFS}/ci/platforms - source orion.sh - cd ${HOMEGFS}/ci/scripts - ./create_experiment.py --yaml ../cases/C48_S2S.yaml --dir ${HOMEGFS} + name: Create Experiments + strategy: + matrix: + pslot: ["C48_S2S"] + steps: + - name: Create Experiment: ${{ matrix.pslot }} + env: + pslot: ${{ matrix.pslot }} + run: | + cd ${{ env.HOMEgfs }} + source workflow/gw_setup.sh + source ci/platforms/orion.sh + ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEgfs }} + if: failure() + uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + with: + labels: "CI-Orion-Running" + uses: actions-ecosystem/action-add-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + with: + labels: "CI-Orion-Failed" run-experiments: needs: create-experiments runs-on: [self-hosted, orion-ready] + strategy: + matrix: + pslot: ["C48_S2S"] steps: - - name: Run Experiment - env: - pslot: C48_S2S - pr: ${{ github.event.pull_request.number }} - EXPDIR: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS/C48_S2S/EXPDIR - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - run: | - cd $HOMEgfs/ci/platforms - source orion.sh - cd $HOMEgfs/ci/scripts - ./run-check_ci.sh + - name: Run Experiment: ${{ matrix.pslot }} + env: + pslot: ${{ matrix.pslot }} + EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR + run: | + cd ${{ env.HOMEgfs }} + source ci/platforms/orion.sh + ./ci/scripts/run-check_ci.sh + if: failure() + uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + with: + labels: "CI-Orion-Running" + uses: actions-ecosystem/action-add-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + with: + labels: "CI-Orion-Failed" + uses: actions-ecosystem/action-create-comment@v1 + if: ${{ startsWith(github.event.comment.body, '/hello') }} + with: + body: | + ${{ env.HOMEgfs}}/global-workflow.log + + + release-lock: + needs: run-experiments + runs-on: [self-hosted, orion-ready] + steps: + - name: Release lock + uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + with: + labels: "CI-Orion-Running" + uses: actions-ecosystem/action-add-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + with: + labels: "CI-Orion-Passed" diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index ee95d8795e..9c05909b50 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -47,12 +47,13 @@ def input_args(): A full path to a YAML file with the following format with required sections: experiment, arguments experiment: - mode: + type: 'gfs' | 'gefs' # TODO: This should be called 'system' not 'type' + mode: 'cycled' | 'forecast-only' used to hold the only required positional argument to setup_expt.py arguments: - holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py - Note: the argument pslot is derived from the basename of the yamlfile itself + holds the remaining key:value pairs for all requisite arguments documented for setup_expt.py + Note: the argument `pslot` is derived from the basename of the yaml file itself Returns ------- @@ -86,7 +87,7 @@ def input_args(): type = setup_expt_args.experiment.type mode = setup_expt_args.experiment.mode - setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) + setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) # TODO:turn setup_expt.py into a function so one does not have to call Executable setup_expt_cmd.add_default_arg(type) setup_expt_cmd.add_default_arg(mode) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index a72db232db..7702b75810 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -3,7 +3,7 @@ set -eux ##################################################################################### # -# Script description: BASH script for checking for cases in a given PR and +# Script description: BASH script for checking for cases in a given PR and # simply running rocotorun on each. This script is intended # to run from within a cron job in the CI Managers account # Abstract TODO @@ -33,7 +33,7 @@ source "${HOMEgfs}/ush/module-setup.sh" module use "${HOMEgfs}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" module list -set -eux +set -x rocotorun=$(command -v rocotorun) if [[ -z ${rocotorun} ]]; then echo "rocotorun not found on system" @@ -56,48 +56,47 @@ else echo "rocotocheck being used from ${rocotocheck}" fi +xml="${EXPDIR}/${pslot}/${pslot}.xml" +db="${EXPDIR}/${pslot}/${pslot}.db" + RUN_COMPLETE="FALSE" while [[ ${RUN_COMPLETE} == "FALSE" ]]; do - xml="${EXPDIR}/${pslot}/${pslot}.xml" - db="${EXPDIR}/${pslot}/${pslot}.db" - if [[ ! -f "${xml}" ]]; then - echo "XML file ${xml} not found, experment ${pslot} failed" - exit 1 + echo "XML file ${xml} not found, experiment ${pslot} failed" + exit 1 fi - echo "Running: ${rocotorun} -v 6 -w ${xml} -d ${db}" + echo "Running: ${rocotorun} -v 10 -w ${xml} -d ${db}" "${rocotorun}" -v 10 -w "${xml}" -d "${db}" # Wait before running rocotostat sleep 60 if [[ ! -f "${db}" ]]; then - echo "Database file ${db} not found, experment ${pslot} failed" - exit 1 + echo "Database file ${db} not found, experiment ${pslot} failed" + exit 1 fi + rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true - echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" || true num_failed=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true + echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" if [[ ${num_failed} -ne 0 ]]; then { echo "Experiment ${pslot} Terminated: *FAILED*" - echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true + echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" } >> "${HOMEgfs}/global-workflow.log" error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true - "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed" { echo "Error logs:" echo "${error_logs}" } >> "${HOMEgfs}/global-workflow.log" sed -i "s/\`\`\`//2g" "${HOMEgfs}/global-workflow.log" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/global-workflow.log" sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "PR\/${pr}\/RUNTESTS" | awk '{print $1}' | xargs scancel || true fi - + if [[ "${num_done}" -eq "${num_cycles}" ]]; then { echo "Experiment ${pslot} completed: *SUCCESS*" @@ -105,8 +104,6 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do echo "with ${num_succeeded} successfully completed jobs" || true } >> "${HOMEgfs}/global-workflow.log" sed -i "s/\`\`\`//2g" "${HOMEgfs}/global-workflow.log" - "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${HOMEgfs}/global-workflow.log" - "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed" RUN_COMPLETE="TRUE" exit 0 fi From 4a668e4d881cd31755e1b0fb47c25404b3b5b259 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 7 Jul 2023 17:36:16 -0500 Subject: [PATCH 100/150] no op to force PR diff --- ci/scripts/run-check_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 7702b75810..0bf9d312f2 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -6,7 +6,7 @@ set -eux # Script description: BASH script for checking for cases in a given PR and # simply running rocotorun on each. This script is intended # to run from within a cron job in the CI Managers account -# Abstract TODO +# Abstract TODO no-op ##################################################################################### scriptname=$(basename "${BASH_SOURCE[0]}") From d16ef39c30504d79add006af94873f80adaf5b76 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 7 Jul 2023 22:41:24 +0000 Subject: [PATCH 101/150] Update globalworkflow-ci.yaml maybe : in step name is a syntax error? --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index b6ef13b6de..bcd5a9191f 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -87,7 +87,7 @@ jobs: matrix: pslot: ["C48_S2S"] steps: - - name: Create Experiment: ${{ matrix.pslot }} + - name: Create Experiment ${{ matrix.pslot }} env: pslot: ${{ matrix.pslot }} run: | From 9efb2c2615860147d03c0929a38cd3cd4b397353 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sat, 8 Jul 2023 20:07:46 +0000 Subject: [PATCH 102/150] seperate actions and run steps for labeling --- .github/workflows/globalworkflow-ci.yaml | 68 ++++++++++++++++++------ 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index bcd5a9191f..35ae1af1d9 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -36,8 +36,12 @@ jobs: run: | cd ${{ env.HOMEgfs }}/sorc ./checkout.sh # Options e.g. -g -u can be added late + if [ $? -ne 0 ]; then + echo "Checkout failed" + exit 1 + fi - - name: Build executables + - needs: Checkout components uses: actions-ecosystem/action-remove-labels@v1 if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: @@ -46,10 +50,25 @@ jobs: if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Building" + + - if ${{ failure() }} + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: "CI-Orion-Ready" + uses: actions-ecosystem/action-add-labels@v1 + with: + labels: "CI-Orion-Failed" + + - name: Build components run: | cd ${{ env.HOMEgfs }}/sorc ./build_all.sh - if: ${{ failure() }} + if [ $? -ne 0 ]; then + echo "Build failed" + exit 1 + fi + + - if ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: @@ -63,7 +82,12 @@ jobs: run: | cd ${{ env.HOMEgfs }}/sorc ./link_workflow.sh - if: success() || failure() + if [ $? -ne 0 ]; then + echo "Linking artifacts failed" + exit 1 + fi + + - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: @@ -73,7 +97,12 @@ jobs: if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Failed" - if: success() + + - needs: Link artifacts + uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + with: + labels: "CI-Orion-Building" uses: actions-ecosystem/action-add-labels@v1 if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: @@ -91,19 +120,24 @@ jobs: env: pslot: ${{ matrix.pslot }} run: | - cd ${{ env.HOMEgfs }} - source workflow/gw_setup.sh - source ci/platforms/orion.sh - ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEgfs }} - if: failure() + cd ${{ env.HOMEgfs }} + source workflow/gw_setup.sh + source ci/platforms/orion.sh + ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEgfs }} + if [ $? -ne 0 ]; then + echo "Create experiment failed" + exit 1 + fi + + - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} - with: - labels: "CI-Orion-Running" + with: + labels: "CI-Orion-Running" uses: actions-ecosystem/action-add-labels@v1 if: ${{ startsWith(github.event.comment.body, '/add-labels') }} - with: - labels: "CI-Orion-Failed" + with: + labels: "CI-Orion-Failed" run-experiments: needs: create-experiments @@ -120,7 +154,12 @@ jobs: cd ${{ env.HOMEgfs }} source ci/platforms/orion.sh ./ci/scripts/run-check_ci.sh - if: failure() + if [ $? -ne 0 ]; then + echo "Run experiment failed" + exit 1 + fi + + if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: @@ -135,7 +174,6 @@ jobs: body: | ${{ env.HOMEgfs}}/global-workflow.log - release-lock: needs: run-experiments runs-on: [self-hosted, orion-ready] From 9a8e7526dbe2eb3df91e2e57736679fb6cec65db Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sat, 8 Jul 2023 21:13:54 +0000 Subject: [PATCH 103/150] Update globalworkflow-ci.yaml separated run and uses steps and added logic for labels accordingly --- .github/workflows/globalworkflow-ci.yaml | 167 +++++++++++------------ 1 file changed, 81 insertions(+), 86 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 35ae1af1d9..09297f79a3 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -40,25 +40,25 @@ jobs: echo "Checkout failed" exit 1 fi - - - needs: Checkout components - uses: actions-ecosystem/action-remove-labels@v1 + + - uses: actions-ecosystem/action-remove-labels@v1 if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Ready" - uses: actions-ecosystem/action-add-labels@v1 + - uses: actions-ecosystem/action-add-labels@v1 if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Building" - - if ${{ failure() }} - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: "CI-Orion-Ready" - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Failed" - + - if: ${{ failure() }} + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: "CI-Orion-Ready" + - if: ${{ failure() }} + uses: actions-ecosystem/action-add-labels@v1 + with: + labels: "CI-Orion-Failed" + - name: Build components run: | cd ${{ env.HOMEgfs }}/sorc @@ -68,13 +68,12 @@ jobs: exit 1 fi - - if ${{ failure() }} + - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Building" + - if: ${{ failure() }} uses: actions-ecosystem/action-add-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Failed" @@ -89,101 +88,97 @@ jobs: - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Building" - if: failure() + + - if: ${{ failure() }} uses: actions-ecosystem/action-add-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Failed" - - needs: Link artifacts - uses: actions-ecosystem/action-remove-labels@v1 + - uses: actions-ecosystem/action-remove-labels@v1 if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Building" - uses: actions-ecosystem/action-add-labels@v1 + + - uses: actions-ecosystem/action-add-labels@v1 if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Running" create-experiments: - needs: checkout-build-link - runs-on: [self-hosted, orion-ready] - name: Create Experiments - strategy: - matrix: - pslot: ["C48_S2S"] - steps: - - name: Create Experiment ${{ matrix.pslot }} - env: - pslot: ${{ matrix.pslot }} - run: | - cd ${{ env.HOMEgfs }} - source workflow/gw_setup.sh - source ci/platforms/orion.sh - ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEgfs }} - if [ $? -ne 0 ]; then - echo "Create experiment failed" - exit 1 - fi - + + needs: checkout-build-link + runs-on: [self-hosted, orion-ready] + strategy: + matrix: + pslot: ["C48_S2S"] + steps: + - name: Create Experiments ${{ matrix.pslot }} + env: + pslot: ${{ matrix.pslot }} + run: | + cd ${{ env.HOMEgfs }} + source workflow/gw_setup.sh + source ci/platforms/orion.sh + ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEgfs }} + if [ $? -ne 0 ]; then + echo "Create experiment failed" + exit 1 + fi + - if: ${{ failure() }} - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + uses: actions-ecosystem/action-remove-labels@v1 with: - labels: "CI-Orion-Running" - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + labels: "CI-Orion-Running" + - if: ${{ failure() }} + uses: actions-ecosystem/action-add-labels@v1 with: - labels: "CI-Orion-Failed" - + labels: "CI-Orion-Failed" + run-experiments: needs: create-experiments runs-on: [self-hosted, orion-ready] - strategy: - matrix: - pslot: ["C48_S2S"] + strategy: + matrix: + pslot: ["C48_S2S"] steps: - - name: Run Experiment: ${{ matrix.pslot }} - env: - pslot: ${{ matrix.pslot }} - EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR - run: | - cd ${{ env.HOMEgfs }} - source ci/platforms/orion.sh - ./ci/scripts/run-check_ci.sh - if [ $? -ne 0 ]; then - echo "Run experiment failed" - exit 1 - fi + - name: Run Experiment ${{ matrix.pslot }} + env: + pslot: ${{ matrix.pslot }} + EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR + run: | + cd ${{ env.HOMEgfs }} + source ci/platforms/orion.sh + ./ci/scripts/run-check_ci.sh + if [ $? -ne 0 ]; then + echo "Run experiment failed" + exit 1 + fi - if: ${{ failure() }} - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} - with: - labels: "CI-Orion-Running" - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/add-labels') }} - with: - labels: "CI-Orion-Failed" - uses: actions-ecosystem/action-create-comment@v1 - if: ${{ startsWith(github.event.comment.body, '/hello') }} - with: - body: | - ${{ env.HOMEgfs}}/global-workflow.log + - if: ${{ failure() }} + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: "CI-Orion-Running" + - if: ${{ failure() }} + uses: actions-ecosystem/action-add-labels@v1 + with: + labels: "CI-Orion-Failed" + - if: ${{ failure() }} + uses: actions-ecosystem/action-create-comment@v1 + with: + body: | + ${{ env.HOMEgfs}}/global-workflow.log release-lock: needs: run-experiments runs-on: [self-hosted, orion-ready] steps: - - name: Release lock - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} - with: - labels: "CI-Orion-Running" - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/add-labels') }} - with: - labels: "CI-Orion-Passed" + - uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + with: + labels: "CI-Orion-Running" + - uses: actions-ecosystem/action-add-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + with: + labels: "CI-Orion-Passed" From 8b235692eab47b2b530c64140a659aac3e061388 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sat, 8 Jul 2023 21:17:36 +0000 Subject: [PATCH 104/150] Update globalworkflow-ci.yaml trest --- .github/workflows/globalworkflow-ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 09297f79a3..927f711c44 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -50,8 +50,9 @@ jobs: with: labels: "CI-Orion-Building" - - if: ${{ failure() }} + - name: foo uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Ready" - if: ${{ failure() }} From 0eda4274d85e7715447117b12a4ec39f9a5eaf06 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sat, 8 Jul 2023 21:34:35 +0000 Subject: [PATCH 105/150] seperate actions and run steps for labeling still --- .github/workflows/globalworkflow-ci.yaml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 927f711c44..c7d9e536fc 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -45,20 +45,25 @@ jobs: if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Ready" + - uses: actions-ecosystem/action-add-labels@v1 if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Building" - - name: foo - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} - with: - labels: "CI-Orion-Ready" - if: ${{ failure() }} uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" + + - name: Link artifacts + run: | + cd ${{ env.HOMEgfs }}/sorc + ./link_workflow.sh + if [ $? -ne 0 ]; then + echo "Linking artifacts failed" + exit 1 + fi - name: Build components run: | @@ -78,15 +83,6 @@ jobs: with: labels: "CI-Orion-Failed" - - name: Link artifacts - run: | - cd ${{ env.HOMEgfs }}/sorc - ./link_workflow.sh - if [ $? -ne 0 ]; then - echo "Linking artifacts failed" - exit 1 - fi - - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 with: From 38b7ea59d8059af238adc0f9f19b745a3de50482 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sun, 9 Jul 2023 17:17:46 +0000 Subject: [PATCH 106/150] Update globalworkflow-ci.yaml removed if on label actions --- .github/workflows/globalworkflow-ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index c7d9e536fc..e309d37750 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -42,12 +42,12 @@ jobs: fi - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + # if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Ready" - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + # if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Building" @@ -94,12 +94,12 @@ jobs: labels: "CI-Orion-Failed" - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} + # if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Building" - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/add-labels') }} + # if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Running" From d611b6afb1a53412b9f13499af433d8012e33f06 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sun, 9 Jul 2023 17:27:23 +0000 Subject: [PATCH 107/150] Update globalworkflow-ci.yaml link was before build - fixed --- .github/workflows/globalworkflow-ci.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index e309d37750..dd7c7ea41c 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -55,22 +55,22 @@ jobs: uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" - - - name: Link artifacts + + - name: Build components run: | cd ${{ env.HOMEgfs }}/sorc - ./link_workflow.sh + ./build_all.sh if [ $? -ne 0 ]; then - echo "Linking artifacts failed" + echo "Build failed" exit 1 fi - - - name: Build components + + - name: Link artifacts run: | cd ${{ env.HOMEgfs }}/sorc - ./build_all.sh + ./link_workflow.sh if [ $? -ne 0 ]; then - echo "Build failed" + echo "Linking artifacts failed" exit 1 fi From 0908ef25fb287a15f3fe5335da13e729f61ce3f4 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sun, 9 Jul 2023 17:45:01 +0000 Subject: [PATCH 108/150] removed if lines on label actions --- .github/workflows/globalworkflow-ci.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index dd7c7ea41c..7f595a1009 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -36,18 +36,12 @@ jobs: run: | cd ${{ env.HOMEgfs }}/sorc ./checkout.sh # Options e.g. -g -u can be added late - if [ $? -ne 0 ]; then - echo "Checkout failed" - exit 1 - fi - uses: actions-ecosystem/action-remove-labels@v1 - # if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Ready" - uses: actions-ecosystem/action-add-labels@v1 - # if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Building" @@ -60,19 +54,11 @@ jobs: run: | cd ${{ env.HOMEgfs }}/sorc ./build_all.sh - if [ $? -ne 0 ]; then - echo "Build failed" - exit 1 - fi - name: Link artifacts run: | cd ${{ env.HOMEgfs }}/sorc ./link_workflow.sh - if [ $? -ne 0 ]; then - echo "Linking artifacts failed" - exit 1 - fi - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 @@ -94,12 +80,10 @@ jobs: labels: "CI-Orion-Failed" - uses: actions-ecosystem/action-remove-labels@v1 - # if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Building" - uses: actions-ecosystem/action-add-labels@v1 - # if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Running" From 6c95102bbf3f175933bff297223d4afa9db64445 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sun, 9 Jul 2023 17:48:48 +0000 Subject: [PATCH 109/150] put || true back in echo for BASH linting --- ci/scripts/run-check_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 0bf9d312f2..ebe866d798 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -86,7 +86,7 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do if [[ ${num_failed} -ne 0 ]]; then { echo "Experiment ${pslot} Terminated: *FAILED*" - echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" + echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true } >> "${HOMEgfs}/global-workflow.log" error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true { From 986971bd9c03b30cb45a01baf408b505d09d611d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 00:52:11 +0000 Subject: [PATCH 110/150] added global env.HOMEgfs into local BASH env and added pr number into run-check BASH env --- .github/workflows/globalworkflow-ci.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 7f595a1009..67c411465a 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -60,15 +60,6 @@ jobs: cd ${{ env.HOMEgfs }}/sorc ./link_workflow.sh - - if: ${{ failure() }} - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: "CI-Orion-Building" - - if: ${{ failure() }} - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Failed" - - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 with: @@ -128,14 +119,12 @@ jobs: env: pslot: ${{ matrix.pslot }} EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR + HOMEgfs: ${{ env.HOMEgfs }} + pr: ${{ github.event.pull_request.number }} run: | - cd ${{ env.HOMEgfs }} + cd $HOMEgfs source ci/platforms/orion.sh ./ci/scripts/run-check_ci.sh - if [ $? -ne 0 ]; then - echo "Run experiment failed" - exit 1 - fi - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 From 51a56c933ebf83f59d9ca5bfceb12ef168f6f50d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 01:28:45 +0000 Subject: [PATCH 111/150] Update globalworkflow-ci.yaml reset HOMEgfs env for run-check --- .github/workflows/globalworkflow-ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 67c411465a..fdfbc55ab5 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -119,9 +119,10 @@ jobs: env: pslot: ${{ matrix.pslot }} EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR - HOMEgfs: ${{ env.HOMEgfs }} + HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} pr: ${{ github.event.pull_request.number }} run: | + echo "HOMEgfs: $HOMEgfs" cd $HOMEgfs source ci/platforms/orion.sh ./ci/scripts/run-check_ci.sh From f1c3a4bfb3a5e8cb58e3a5eb4589827091b13c91 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 13:14:59 +0000 Subject: [PATCH 112/150] Had to work around HOMEgfs not being an env var in BASH shell that runs setup_xml --- .github/workflows/globalworkflow-ci.yaml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index fdfbc55ab5..b110c74c30 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -13,7 +13,7 @@ on: - labeled env: - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} RUNTESTS: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS MACHINE_ID: orion @@ -32,7 +32,7 @@ jobs: - name: Checkout components env: - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} + HOMEgfs: ${{ env.HOMEGFS }} run: | cd ${{ env.HOMEgfs }}/sorc ./checkout.sh # Options e.g. -g -u can be added late @@ -51,13 +51,17 @@ jobs: labels: "CI-Orion-Failed" - name: Build components + env: + HOMEgfs: ${{ env.HOMEGFS }} run: | - cd ${{ env.HOMEgfs }}/sorc + cd ${HOMEgfs}/sorc ./build_all.sh - name: Link artifacts + env: + HOMEgfs: ${{ env.HOMEGFS }} run: | - cd ${{ env.HOMEgfs }}/sorc + cd ${HOMEgfs}/sorc ./link_workflow.sh - if: ${{ failure() }} @@ -85,15 +89,16 @@ jobs: strategy: matrix: pslot: ["C48_S2S"] + steps: - name: Create Experiments ${{ matrix.pslot }} env: pslot: ${{ matrix.pslot }} run: | - cd ${{ env.HOMEgfs }} + cd ${{ env.HOMEGFS }} source workflow/gw_setup.sh source ci/platforms/orion.sh - ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEgfs }} + ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEGFS }} if [ $? -ne 0 ]; then echo "Create experiment failed" exit 1 @@ -117,12 +122,11 @@ jobs: steps: - name: Run Experiment ${{ matrix.pslot }} env: + HOMEgfs: $${{ env.HOMEGFS }} pslot: ${{ matrix.pslot }} EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR - HOMEgfs: ${{ github.workspace }}/${{ github.event.pull_request.number }} pr: ${{ github.event.pull_request.number }} run: | - echo "HOMEgfs: $HOMEgfs" cd $HOMEgfs source ci/platforms/orion.sh ./ci/scripts/run-check_ci.sh From 955df5ef8beb2a9962347728e817d4590ebec536 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 13:51:19 +0000 Subject: [PATCH 113/150] Update globalworkflow-ci.yaml asked for TOKEN at the end, suggested to use pull_request_target for more PR access --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index b110c74c30..157c7ba8f0 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -8,7 +8,7 @@ name: gw-ci-orion # - CI-Orion-Run on: - pull_request: + pull_request_target: types: - labeled From e44f1d3324f9f1cd5745df917dfddb93f19f8f96 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 13:56:06 +0000 Subject: [PATCH 114/150] the access token error is a GitHub glitch, juste needed to re-run --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 157c7ba8f0..b110c74c30 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -8,7 +8,7 @@ name: gw-ci-orion # - CI-Orion-Run on: - pull_request_target: + pull_request: types: - labeled From 428eb493b32275b134bd8cb1a619eec084ced7f9 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 14:31:17 +0000 Subject: [PATCH 115/150] Update globalworkflow-ci.yaml create comment needed github_token --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index b110c74c30..76760a9f6c 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -142,6 +142,7 @@ jobs: - if: ${{ failure() }} uses: actions-ecosystem/action-create-comment@v1 with: + github_token: ${{ secrets.github_token }} body: | ${{ env.HOMEgfs}}/global-workflow.log From 717b3647321d1fc669c69c447d5459e8d72d15dc Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 15:19:54 +0000 Subject: [PATCH 116/150] extra dollor sign env HOMEgfs re-assiginment and PR no longer in path for sacct/scancel line --- .github/workflows/globalworkflow-ci.yaml | 2 +- ci/scripts/run-check_ci.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 76760a9f6c..6c902e1dbb 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -122,7 +122,7 @@ jobs: steps: - name: Run Experiment ${{ matrix.pslot }} env: - HOMEgfs: $${{ env.HOMEGFS }} + HOMEgfs: ${{ env.HOMEGFS }} pslot: ${{ matrix.pslot }} EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR pr: ${{ github.event.pull_request.number }} diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index ebe866d798..8aa1a61c54 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -94,7 +94,7 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do echo "${error_logs}" } >> "${HOMEgfs}/global-workflow.log" sed -i "s/\`\`\`//2g" "${HOMEgfs}/global-workflow.log" - sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "PR\/${pr}\/RUNTESTS" | awk '{print $1}' | xargs scancel || true + sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "${pr}\/RUNTESTS" | awk '{print $1}' | xargs scancel || true fi if [[ "${num_done}" -eq "${num_cycles}" ]]; then From b6a9e8649de22d105381f7dd8d1989beb91bb248 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 16:04:46 +0000 Subject: [PATCH 117/150] added second test case via matrix systax --- .github/workflows/globalworkflow-ci.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 6c902e1dbb..e76e6a9293 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -1,11 +1,6 @@ name: gw-ci-orion # A GitHub Action that creates a deployment for a PR, clones, builds and runs a test suite -# This workflow is triggered by a label being added to a PR -# The label name determines the environment to deploy to -# The label name must be one of the following: -# - CI-Orion-Ready -# - CI-Orion-Build -# - CI-Orion-Run +# This workflow is triggered by a label being added to a PR name Orion-Ready on: pull_request: @@ -88,7 +83,7 @@ jobs: runs-on: [self-hosted, orion-ready] strategy: matrix: - pslot: ["C48_S2S"] + pslot: ["C48_S2S","C96_atm3DVar"] steps: - name: Create Experiments ${{ matrix.pslot }} @@ -118,7 +113,7 @@ jobs: runs-on: [self-hosted, orion-ready] strategy: matrix: - pslot: ["C48_S2S"] + pslot: ["C48_S2S","C96_atm3DVar"] steps: - name: Run Experiment ${{ matrix.pslot }} env: From e3b7626e81278ec129f5765fd42d3fd948dceb95 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 16:24:58 +0000 Subject: [PATCH 118/150] removed example conditionals on labeling Passed in release-lock --- .github/workflows/globalworkflow-ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index e76e6a9293..0731b1ebd0 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -146,10 +146,8 @@ jobs: runs-on: [self-hosted, orion-ready] steps: - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/remove-labels') }} with: labels: "CI-Orion-Running" - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ startsWith(github.event.comment.body, '/add-labels') }} with: labels: "CI-Orion-Passed" From d4bf5427217c8d841f8d68bb0aa5672ab9aff9b8 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 16:27:13 +0000 Subject: [PATCH 119/150] Update run-check_ci.sh removed old no op --- ci/scripts/run-check_ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 8aa1a61c54..91627a8b57 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -6,7 +6,6 @@ set -eux # Script description: BASH script for checking for cases in a given PR and # simply running rocotorun on each. This script is intended # to run from within a cron job in the CI Managers account -# Abstract TODO no-op ##################################################################################### scriptname=$(basename "${BASH_SOURCE[0]}") From 1d37abb1ae5d365268db4d6d786171a2a7f72c22 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 17:33:55 +0000 Subject: [PATCH 120/150] reduced time to wait between rocoto pings --- ci/scripts/run-check_ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 8aa1a61c54..1a783ef0f1 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -70,7 +70,7 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do "${rocotorun}" -v 10 -w "${xml}" -d "${db}" # Wait before running rocotostat - sleep 60 + sleep 30 if [[ ! -f "${db}" ]]; then echo "Database file ${db} not found, experiment ${pslot} failed" exit 1 @@ -109,6 +109,6 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do fi # Wait before running rocotorun again - sleep 240 + sleep 210 done From 3a5d137742c2c31fd2e8e3aee5650215865fee98 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Mon, 10 Jul 2023 12:44:38 -0500 Subject: [PATCH 121/150] removed two cases for testing --- ci/cases/C96C48_hybatmDA.yaml | 17 ----------------- ci/cases/C96_atm3DVar.yaml | 16 ---------------- 2 files changed, 33 deletions(-) delete mode 100644 ci/cases/C96C48_hybatmDA.yaml delete mode 100644 ci/cases/C96_atm3DVar.yaml diff --git a/ci/cases/C96C48_hybatmDA.yaml b/ci/cases/C96C48_hybatmDA.yaml deleted file mode 100644 index 6a0e41fbe1..0000000000 --- a/ci/cases/C96C48_hybatmDA.yaml +++ /dev/null @@ -1,17 +0,0 @@ -experiment: - type: gfs - mode: cycled - -arguments: - app: ATM - resdet: 96 - resens: 48 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR - icsdir: ${ICSDIR_ROOT}/C96C48 - idate: 2021122018 - edate: 2021122106 - nens: 2 - gfs_cyc: 1 - start: cold - yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml diff --git a/ci/cases/C96_atm3DVar.yaml b/ci/cases/C96_atm3DVar.yaml deleted file mode 100644 index 5a93bd6454..0000000000 --- a/ci/cases/C96_atm3DVar.yaml +++ /dev/null @@ -1,16 +0,0 @@ -experiment: - type: gfs - mode: cycled - -arguments: - app: ATM - resdet: 96 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR - icsdir: ${ICSDIR_ROOT}/C96C48 - idate: 2021122018 - edate: 2021122106 - nens: 0 - gfs_cyc: 1 - start: cold - yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml From f3b640336433a0e44415614f9287cfbccf4b1474 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Mon, 10 Jul 2023 18:02:58 +0000 Subject: [PATCH 122/150] added switch to create experment to tag pslot name --- ci/scripts/create_experiment.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index c8b95aaf0e..bac5f4fa25 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -20,16 +20,18 @@ with an error code of 0 upon success. """ -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from pathlib import Path -from wxflow import YAMLFile, Logger, logit, Executable +from pygw.yaml_file import YAMLFile +from pygw.logger import Logger +from pygw.executable import Executable + +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter logger = Logger(level='DEBUG', colored_log=True) -@logit(logger) def input_args(): """ Method to collect user arguments for `create_experiment.py` @@ -45,12 +47,13 @@ def input_args(): A full path to a YAML file with the following format with required sections: experiment, arguments experiment: - mode: + type: 'gfs' | 'gefs' # TODO: This should be called 'system' not 'type' + mode: 'cycled' | 'forecast-only' used to hold the only required positional argument to setup_expt.py arguments: - holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py - Note: the argument pslot is derived from the basename of the yamlfile itself + holds the remaining key:value pairs for all requisite arguments documented for setup_expt.py + Note: the argument `pslot` is derived from the basename of the yaml file itself Returns ------- @@ -69,6 +72,7 @@ def input_args(): parser.add_argument('--yaml', help='yaml configuration file per experiment', type=str, required=True) parser.add_argument('--dir', help='full path to top level of repo of global-workflow', type=str, required=True) + parser.add_argument('--pslot_tag', help='add a tag to pslot name for uniqueness', type=str, required=False) args = parser.parse_args() return args @@ -80,11 +84,11 @@ def input_args(): setup_expt_args = YAMLFile(path=user_inputs.yaml) HOMEgfs = Path.absolute(Path(user_inputs.dir)) - pslot = Path(user_inputs.yaml).stem + pslot = Path(user_inputs.yaml).stem+user_inputs.pslot_tag type = setup_expt_args.experiment.type mode = setup_expt_args.experiment.mode - setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) + setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) # TODO:turn setup_expt.py into a function so one does not have to call Executable setup_expt_cmd.add_default_arg(type) setup_expt_cmd.add_default_arg(mode) From a1aaf6e5ba4097c799996672b47a3fadfb263a11 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Mon, 10 Jul 2023 13:43:38 -0500 Subject: [PATCH 123/150] updates for adding has to pslot name --- ci/scripts/create_experiment.py | 21 +++++++++------------ ci/scripts/driver.sh | 4 ++++ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index bac5f4fa25..9ba4f00fc5 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -20,18 +20,17 @@ with an error code of 0 upon success. """ +import os +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from pathlib import Path -from pygw.yaml_file import YAMLFile -from pygw.logger import Logger -from pygw.executable import Executable - -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +from wxflow import YAMLFile, Logger, logit, Executable logger = Logger(level='DEBUG', colored_log=True) +@logit(logger) def input_args(): """ Method to collect user arguments for `create_experiment.py` @@ -47,13 +46,12 @@ def input_args(): A full path to a YAML file with the following format with required sections: experiment, arguments experiment: - type: 'gfs' | 'gefs' # TODO: This should be called 'system' not 'type' - mode: 'cycled' | 'forecast-only' + mode: used to hold the only required positional argument to setup_expt.py arguments: - holds the remaining key:value pairs for all requisite arguments documented for setup_expt.py - Note: the argument `pslot` is derived from the basename of the yaml file itself + holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py + Note: the argument pslot is derived from the basename of the yamlfile itself Returns ------- @@ -72,7 +70,6 @@ def input_args(): parser.add_argument('--yaml', help='yaml configuration file per experiment', type=str, required=True) parser.add_argument('--dir', help='full path to top level of repo of global-workflow', type=str, required=True) - parser.add_argument('--pslot_tag', help='add a tag to pslot name for uniqueness', type=str, required=False) args = parser.parse_args() return args @@ -84,11 +81,11 @@ def input_args(): setup_expt_args = YAMLFile(path=user_inputs.yaml) HOMEgfs = Path.absolute(Path(user_inputs.dir)) - pslot = Path(user_inputs.yaml).stem+user_inputs.pslot_tag + pslot = os.environ['pslot'] type = setup_expt_args.experiment.type mode = setup_expt_args.experiment.mode - setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) # TODO:turn setup_expt.py into a function so one does not have to call Executable + setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) setup_expt_cmd.add_default_arg(type) setup_expt_cmd.add_default_arg(mode) diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index 1bf77537fa..d2290adc7d 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -148,8 +148,12 @@ for pr in ${pr_list}; do ############################################################# HOMEgfs_PR="${pr_dir}/global-workflow" export HOMEgfs_PR + cd "${HOMEgfs_PR}" + discribe=$(git rev-parse --short HEAD) + for yaml_config in "${HOMEgfs_PR}/ci/cases/"*.yaml; do pslot=$(basename "${yaml_config}" .yaml) || true + pslot="${pslot}_${discribe}" export pslot set +e "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${pslot}.yaml" --dir "${HOMEgfs_PR}" From 46e2935edcee5e4d9fe32a8c1471206e63634cf2 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 11 Jul 2023 17:18:25 +0000 Subject: [PATCH 124/150] Update globalworkflow-ci.yaml added HASH to pslot name --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 0731b1ebd0..cb0f205ee1 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -88,7 +88,7 @@ jobs: steps: - name: Create Experiments ${{ matrix.pslot }} env: - pslot: ${{ matrix.pslot }} + pslot: ${{ matrix.pslot }}_${{ github.event.pull_request.head.sha }} run: | cd ${{ env.HOMEGFS }} source workflow/gw_setup.sh @@ -118,7 +118,7 @@ jobs: - name: Run Experiment ${{ matrix.pslot }} env: HOMEgfs: ${{ env.HOMEGFS }} - pslot: ${{ matrix.pslot }} + pslot: ${{ matrix.pslot }}_${{ github.event.pull_request.head.sha }} EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR pr: ${{ github.event.pull_request.number }} run: | From 115c7995e18772175a6e4b4b82d5fb64360ef2cd Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 11 Jul 2023 13:40:29 -0500 Subject: [PATCH 125/150] fine tuning pslot with hash name --- ci/cases/C48_S2S.yaml | 5 +++-- ci/scripts/create_experiment.py | 6 +----- ci/scripts/driver.sh | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ci/cases/C48_S2S.yaml b/ci/cases/C48_S2S.yaml index c1c59b5659..812ea42d75 100644 --- a/ci/cases/C48_S2S.yaml +++ b/ci/cases/C48_S2S.yaml @@ -3,10 +3,11 @@ experiment: mode: forecast-only arguments: + pslot ${pslot} app: S2S resdet: 48 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR + comrot: ${RUNTESTS}/COMROT + expdir: ${RUNTESTS}/EXPDIR idate: 2021032312 edate: 2021032312 yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index 9ba4f00fc5..1d789840e4 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -81,7 +81,6 @@ def input_args(): setup_expt_args = YAMLFile(path=user_inputs.yaml) HOMEgfs = Path.absolute(Path(user_inputs.dir)) - pslot = os.environ['pslot'] type = setup_expt_args.experiment.type mode = setup_expt_args.experiment.mode @@ -94,9 +93,6 @@ def input_args(): setup_expt_cmd.add_default_arg(f'--{conf}') setup_expt_cmd.add_default_arg(str(value)) - setup_expt_cmd.add_default_arg('--pslot') - setup_expt_cmd.add_default_arg(pslot) - logger.info(f'Run command: {setup_expt_cmd.command}') setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr')) setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout')) @@ -104,7 +100,7 @@ def input_args(): setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr) setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py')) - expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(pslot))) + expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir))) setup_xml_cmd.add_default_arg(str(expdir)) logger.info(f'Run command: {setup_xml_cmd.command}') diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index d2290adc7d..728b0d5cd9 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -152,11 +152,11 @@ for pr in ${pr_list}; do discribe=$(git rev-parse --short HEAD) for yaml_config in "${HOMEgfs_PR}/ci/cases/"*.yaml; do - pslot=$(basename "${yaml_config}" .yaml) || true + case=$(basename "${yaml_config}" .yaml) || true pslot="${pslot}_${discribe}" export pslot set +e - "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${pslot}.yaml" --dir "${HOMEgfs_PR}" + "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml" --dir "${HOMEgfs_PR}" ci_status=$? set -e if [[ ${ci_status} -eq 0 ]]; then From 46afc357e23165ace4522f5125cc15ea07757e7b Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 11 Jul 2023 18:57:10 +0000 Subject: [PATCH 126/150] put pslot into YAML case file and updated path to EXPDIR --- ci/scripts/create_experiment.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index 738d5c1ea6..1d789840e4 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -20,6 +20,7 @@ with an error code of 0 upon success. """ +import os from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from pathlib import Path @@ -45,13 +46,12 @@ def input_args(): A full path to a YAML file with the following format with required sections: experiment, arguments experiment: - type: 'gfs' | 'gefs' # TODO: This should be called 'system' not 'type' - mode: 'cycled' | 'forecast-only' + mode: used to hold the only required positional argument to setup_expt.py arguments: - holds the remaining key:value pairs for all requisite arguments documented for setup_expt.py - Note: the argument `pslot` is derived from the basename of the yaml file itself + holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py + Note: the argument pslot is derived from the basename of the yamlfile itself Returns ------- @@ -81,11 +81,10 @@ def input_args(): setup_expt_args = YAMLFile(path=user_inputs.yaml) HOMEgfs = Path.absolute(Path(user_inputs.dir)) - pslot = Path(user_inputs.yaml).stem type = setup_expt_args.experiment.type mode = setup_expt_args.experiment.mode - setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) # TODO:turn setup_expt.py into a function so one does not have to call Executable + setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) setup_expt_cmd.add_default_arg(type) setup_expt_cmd.add_default_arg(mode) @@ -94,9 +93,6 @@ def input_args(): setup_expt_cmd.add_default_arg(f'--{conf}') setup_expt_cmd.add_default_arg(str(value)) - setup_expt_cmd.add_default_arg('--pslot') - setup_expt_cmd.add_default_arg(pslot) - logger.info(f'Run command: {setup_expt_cmd.command}') setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr')) setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout')) @@ -104,7 +100,7 @@ def input_args(): setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr) setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py')) - expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(pslot))) + expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir))) setup_xml_cmd.add_default_arg(str(expdir)) logger.info(f'Run command: {setup_xml_cmd.command}') From 84a70e797691504aa7ef39120ddc753d0bdbfe34 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 11 Jul 2023 19:08:23 +0000 Subject: [PATCH 127/150] Update create_experiment.py removed special handling of pslot in create_experments --- ci/scripts/create_experiment.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index 738d5c1ea6..5d8466dabc 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -81,7 +81,6 @@ def input_args(): setup_expt_args = YAMLFile(path=user_inputs.yaml) HOMEgfs = Path.absolute(Path(user_inputs.dir)) - pslot = Path(user_inputs.yaml).stem type = setup_expt_args.experiment.type mode = setup_expt_args.experiment.mode @@ -100,11 +99,10 @@ def input_args(): logger.info(f'Run command: {setup_expt_cmd.command}') setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr')) setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout')) - print(setup_expt_stderr) setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr) setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py')) - expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(pslot))) + expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir))) setup_xml_cmd.add_default_arg(str(expdir)) logger.info(f'Run command: {setup_xml_cmd.command}') From ccf7ad52c18ba2fa4842173f3adb7fcb3fbef216 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Tue, 11 Jul 2023 14:18:22 -0500 Subject: [PATCH 128/150] moved pslot into env in run shell and added hash tag --- .github/workflows/globalworkflow-ci.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index cb0f205ee1..41200b5a77 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -87,17 +87,12 @@ jobs: steps: - name: Create Experiments ${{ matrix.pslot }} - env: - pslot: ${{ matrix.pslot }}_${{ github.event.pull_request.head.sha }} run: | cd ${{ env.HOMEGFS }} source workflow/gw_setup.sh source ci/platforms/orion.sh + export pslot="${{ matrix.pslot }}/_${LAST_COMMIT_SHA:0:5}" ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEGFS }} - if [ $? -ne 0 ]; then - echo "Create experiment failed" - exit 1 - fi - if: ${{ failure() }} uses: actions-ecosystem/action-remove-labels@v1 @@ -118,12 +113,12 @@ jobs: - name: Run Experiment ${{ matrix.pslot }} env: HOMEgfs: ${{ env.HOMEGFS }} - pslot: ${{ matrix.pslot }}_${{ github.event.pull_request.head.sha }} EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR pr: ${{ github.event.pull_request.number }} run: | cd $HOMEgfs source ci/platforms/orion.sh + pslot="${{ matrix.pslot }}/_${LAST_COMMIT_SHA:0:5}" ./ci/scripts/run-check_ci.sh - if: ${{ failure() }} From 68c63ec0d4170ddcf833573dee8c20a9c42c24be Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 11 Jul 2023 20:42:25 +0000 Subject: [PATCH 129/150] added second case for testing --- ci/cases/C96_atm3DVar.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 ci/cases/C96_atm3DVar.yaml diff --git a/ci/cases/C96_atm3DVar.yaml b/ci/cases/C96_atm3DVar.yaml new file mode 100644 index 0000000000..f103f08d1f --- /dev/null +++ b/ci/cases/C96_atm3DVar.yaml @@ -0,0 +1,17 @@ +experiment: + type: gfs + mode: cycled + +arguments: + pslot: ${pslot} + app: ATM + resdet: 96 + comrot: ${RUNTESTS}/COMROT + expdir: ${RUNTESTS}/EXPDIR + icsdir: ${ICSDIR_ROOT}/C96C48 + idate: 2021122018 + edate: 2021122106 + nens: 0 + gfs_cyc: 1 + start: cold + yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml From edeae69d5b8aad4e428d0f564ad359cd56473889 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 11 Jul 2023 20:43:21 +0000 Subject: [PATCH 130/150] : again in C48 for pslot --- ci/cases/C48_S2S.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/C48_S2S.yaml b/ci/cases/C48_S2S.yaml index 812ea42d75..00d391559d 100644 --- a/ci/cases/C48_S2S.yaml +++ b/ci/cases/C48_S2S.yaml @@ -3,7 +3,7 @@ experiment: mode: forecast-only arguments: - pslot ${pslot} + pslot: ${pslot} app: S2S resdet: 48 comrot: ${RUNTESTS}/COMROT From f43b6e9ec8ba780767cbaeddae2d4a18dc740dd6 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 11 Jul 2023 20:57:48 +0000 Subject: [PATCH 131/150] used run.id intead o HASH --- .github/workflows/globalworkflow-ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 41200b5a77..ed29a27198 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -87,11 +87,12 @@ jobs: steps: - name: Create Experiments ${{ matrix.pslot }} + env: + pslot: ${{ matrix.pslot }}_${{ github.run.id }} run: | cd ${{ env.HOMEGFS }} source workflow/gw_setup.sh source ci/platforms/orion.sh - export pslot="${{ matrix.pslot }}/_${LAST_COMMIT_SHA:0:5}" ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEGFS }} - if: ${{ failure() }} @@ -114,11 +115,11 @@ jobs: env: HOMEgfs: ${{ env.HOMEGFS }} EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR + pslot: ${{ matrix.pslot }}_${{ github.run.id }} pr: ${{ github.event.pull_request.number }} run: | cd $HOMEgfs source ci/platforms/orion.sh - pslot="${{ matrix.pslot }}/_${LAST_COMMIT_SHA:0:5}" ./ci/scripts/run-check_ci.sh - if: ${{ failure() }} From 72fa5394dc5805a8eab0ac2ff45d7b000f1b6dc0 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 11 Jul 2023 16:58:47 -0400 Subject: [PATCH 132/150] tighten the runner dirs --- .github/workflows/globalworkflow-ci.yaml | 92 +++++++++++------------- ci/cases/C48_S2S.yaml | 4 +- ci/cases/C96C48_hybatmDA.yaml | 4 +- ci/cases/C96_atm3DVar.yaml | 4 +- ci/scripts/run-check_ci.sh | 76 +++++--------------- 5 files changed, 66 insertions(+), 114 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 41200b5a77..16761dd8d8 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -2,20 +2,16 @@ name: gw-ci-orion # A GitHub Action that creates a deployment for a PR, clones, builds and runs a test suite # This workflow is triggered by a label being added to a PR name Orion-Ready -on: - pull_request: - types: - - labeled +on: [workflow_dispatch] env: - HOMEGFS: ${{ github.workspace }}/${{ github.event.pull_request.number }} HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} - RUNTESTS: ${{ github.workspace }}/${{ github.event.pull_request.number }}/RUNTESTS + RUNTESTS: ${{ github.workspace }}/RUNTESTS/${{ github.event.pull_request.number }} MACHINE_ID: orion jobs: checkout-build-link: - if: github.event.label.name == 'CI-Orion-Ready' + if: github.event.label.name == 'CI-Orion-Ready' runs-on: [self-hosted, orion-ready] timeout-minutes: 600 @@ -26,110 +22,104 @@ jobs: path: ${{ github.event.pull_request.number }} - name: Checkout components - env: - HOMEgfs: ${{ env.HOMEGFS }} run: | - cd ${{ env.HOMEgfs }}/sorc + cd ${{ env.HOMEgfs_PR }}/sorc ./checkout.sh # Options e.g. -g -u can be added late - - - uses: actions-ecosystem/action-remove-labels@v1 + + - if: success() or failure() + uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Ready" - - uses: actions-ecosystem/action-add-labels@v1 + - if: success() + uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Building" - - if: ${{ failure() }} + - if: failure() uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" - + - name: Build components - env: - HOMEgfs: ${{ env.HOMEGFS }} run: | - cd ${HOMEgfs}/sorc + cd ${{ env.HOMEgfs_PR }}/sorc ./build_all.sh - - - name: Link artifacts - env: - HOMEgfs: ${{ env.HOMEGFS }} - run: | - cd ${HOMEgfs}/sorc - ./link_workflow.sh - - if: ${{ failure() }} + - if: success() or failure() uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Building" - - - if: ${{ failure() }} + + - if: failure() uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" - - uses: actions-ecosystem/action-remove-labels@v1 + - name: Link artifacts + run: | + cd ${{ env.HOMEgfs_PR }}/sorc + ./link_workflow.sh + + - if: failure() + uses: actions-ecosystem/action-add-labels@v1 with: - labels: "CI-Orion-Building" - - - uses: actions-ecosystem/action-add-labels@v1 + labels: "CI-Orion-Failed" + + - if: success() + uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Running" create-experiments: - needs: checkout-build-link runs-on: [self-hosted, orion-ready] strategy: matrix: - pslot: ["C48_S2S","C96_atm3DVar"] + case: ["C48_S2S","C96_atm3DVar"] steps: - - name: Create Experiments ${{ matrix.pslot }} + - name: Create Experiments ${{ matrix.case }} run: | - cd ${{ env.HOMEGFS }} + cd ${{ env.HOMEgfs_PR }} source workflow/gw_setup.sh source ci/platforms/orion.sh - export pslot="${{ matrix.pslot }}/_${LAST_COMMIT_SHA:0:5}" - ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.pslot }}.yaml --dir ${{ env.HOMEGFS }} - - - if: ${{ failure() }} + export pslot="${{ matrix.case }}.${GITHUB_SHA::5}" + ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }} + + - if: failure() uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Running" - - if: ${{ failure() }} uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" - + run-experiments: needs: create-experiments runs-on: [self-hosted, orion-ready] strategy: matrix: - pslot: ["C48_S2S","C96_atm3DVar"] + case: ["C48_S2S","C96_atm3DVar"] steps: - - name: Run Experiment ${{ matrix.pslot }} + - name: Run Experiment ${{ matrix.case }} env: - HOMEgfs: ${{ env.HOMEGFS }} - EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR + HOMEgfs: ${{ env.HOMEgfs_PR }} + EXPDIR: ${{ env.RUNTESTS }}/EXPDIR pr: ${{ github.event.pull_request.number }} run: | - cd $HOMEgfs + cd ${HOMEgfs} source ci/platforms/orion.sh - pslot="${{ matrix.pslot }}/_${LAST_COMMIT_SHA:0:5}" + pslot="${{ matrix.case }}.${GITHUB_SHA::5}" ./ci/scripts/run-check_ci.sh - - if: ${{ failure() }} + - if: failure() uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Running" - - if: ${{ failure() }} uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" - - if: ${{ failure() }} uses: actions-ecosystem/action-create-comment@v1 with: github_token: ${{ secrets.github_token }} diff --git a/ci/cases/C48_S2S.yaml b/ci/cases/C48_S2S.yaml index c1c59b5659..25c12225e7 100644 --- a/ci/cases/C48_S2S.yaml +++ b/ci/cases/C48_S2S.yaml @@ -5,8 +5,8 @@ experiment: arguments: app: S2S resdet: 48 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR + comrot: ${RUNTESTS}/COMROT + expdir: ${RUNTESTS}/EXPDIR idate: 2021032312 edate: 2021032312 yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml diff --git a/ci/cases/C96C48_hybatmDA.yaml b/ci/cases/C96C48_hybatmDA.yaml index 6a0e41fbe1..cbb29e7128 100644 --- a/ci/cases/C96C48_hybatmDA.yaml +++ b/ci/cases/C96C48_hybatmDA.yaml @@ -6,8 +6,8 @@ arguments: app: ATM resdet: 96 resens: 48 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR + comrot: ${RUNTESTS}/COMROT + expdir: ${RUNTESTS}/EXPDIR icsdir: ${ICSDIR_ROOT}/C96C48 idate: 2021122018 edate: 2021122106 diff --git a/ci/cases/C96_atm3DVar.yaml b/ci/cases/C96_atm3DVar.yaml index 5a93bd6454..68d5aa022d 100644 --- a/ci/cases/C96_atm3DVar.yaml +++ b/ci/cases/C96_atm3DVar.yaml @@ -5,8 +5,8 @@ experiment: arguments: app: ATM resdet: 96 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR + comrot: ${RUNTESTS}/COMROT + expdir: ${RUNTESTS}/EXPDIR icsdir: ${ICSDIR_ROOT}/C96C48 idate: 2021122018 edate: 2021122106 diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index bac40d06e8..5c4332d99e 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -eux ##################################################################################### # @@ -8,63 +7,21 @@ set -eux # to run from within a cron job in the CI Managers account ##################################################################################### -scriptname=$(basename "${BASH_SOURCE[0]}") -echo "Begin ${scriptname} at $(date -u)" || true -export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/workflow/gw_setup.sh" -######################################################################### -# Set up runtime environment varibles for accounts on supproted machines -######################################################################### +xml="${EXPDIR}/${pslot}/${pslot}.xml" +db="${EXPDIR}/${pslot}/${pslot}.db" -source "${HOMEgfs}/ush/detect_machine.sh" -case ${MACHINE_ID} in - hera | orion) - echo "Running Automated Testing on ${MACHINE_ID}" - source "${HOMEgfs}/ci/platforms/${MACHINE_ID}.sh" - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; -esac -set +x -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/modulefiles" -module load "module_gwsetup.${MACHINE_ID}" -module list -set -x -rocotorun=$(command -v rocotorun) -if [[ -z ${rocotorun} ]]; then - echo "rocotorun not found on system" - exit 1 -else - echo "rocotorun being used from ${rocotorun}" -fi -rocotostat=$(command -v rocotostat) -if [[ -z ${rocotostat+x} ]]; then - echo "rocotostat not found on system" +# Ensure the XML is present for the experiment +if [[ ! -f "${xml}" ]]; then + echo "XML file ${xml} not found, experiment ${pslot} failed" exit 1 -else - echo "rocotostat being used from ${rocotostat}" -fi -rocotocheck=$(command -v rocotocheck) -if [[ -z ${rocotocheck+x} ]]; then - echo "rocotocheck not found on system" - exit 1 -else - echo "rocotocheck being used from ${rocotocheck}" fi -xml="${EXPDIR}/${pslot}/${pslot}.xml" -db="${EXPDIR}/${pslot}/${pslot}.db" - -RUN_COMPLETE="FALSE" -while [[ ${RUN_COMPLETE} == "FALSE" ]]; do +rc=99 +while true; do - if [[ ! -f "${xml}" ]]; then - echo "XML file ${xml} not found, experiment ${pslot} failed" - exit 1 - fi echo "Running: ${rocotorun} -v 10 -w ${xml} -d ${db}" "${rocotorun}" -v 10 -w "${xml}" -d "${db}" @@ -72,14 +29,16 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do sleep 30 if [[ ! -f "${db}" ]]; then echo "Database file ${db} not found, experiment ${pslot} failed" - exit 1 + exit 2 fi + # Get job statistics rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true num_failed=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true + echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" if [[ ${num_failed} -ne 0 ]]; then @@ -87,6 +46,7 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do echo "Experiment ${pslot} Terminated: *FAILED*" echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true } >> "${HOMEgfs}/global-workflow.log" + error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true { echo "Error logs:" @@ -96,18 +56,20 @@ while [[ ${RUN_COMPLETE} == "FALSE" ]]; do sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "${pr}\/RUNTESTS" | awk '{print $1}' | xargs scancel || true fi - if [[ "${num_done}" -eq "${num_cycles}" ]]; then + if [[ "${num_done}" -eq "${num_cycles}" ]]; then { echo "Experiment ${pslot} completed: *SUCCESS*" echo "Experiment ${pslot} Completed at $(date)" || true echo "with ${num_succeeded} successfully completed jobs" || true } >> "${HOMEgfs}/global-workflow.log" sed -i "s/\`\`\`//2g" "${HOMEgfs}/global-workflow.log" - RUN_COMPLETE="TRUE" - exit 0 + rc=0 + break fi # Wait before running rocotorun again - sleep 210 + sleep 300 done + +exit "${rc}" From 41bec08695eb9e15058aad841f03adebce8ad51b Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 11 Jul 2023 21:00:06 +0000 Subject: [PATCH 133/150] Update globalworkflow-ci.yaml indentation worng --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index ed29a27198..c9582ac71c 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -87,7 +87,7 @@ jobs: steps: - name: Create Experiments ${{ matrix.pslot }} - env: + env: pslot: ${{ matrix.pslot }}_${{ github.run.id }} run: | cd ${{ env.HOMEGFS }} From 7fd81d4591c34deb1dec64d4e22b67f56cc9230a Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 11 Jul 2023 17:03:37 -0400 Subject: [PATCH 134/150] remove deleted file --- ci/cases/C96C48_hybatmDA.yaml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 ci/cases/C96C48_hybatmDA.yaml diff --git a/ci/cases/C96C48_hybatmDA.yaml b/ci/cases/C96C48_hybatmDA.yaml deleted file mode 100644 index cbb29e7128..0000000000 --- a/ci/cases/C96C48_hybatmDA.yaml +++ /dev/null @@ -1,17 +0,0 @@ -experiment: - type: gfs - mode: cycled - -arguments: - app: ATM - resdet: 96 - resens: 48 - comrot: ${RUNTESTS}/COMROT - expdir: ${RUNTESTS}/EXPDIR - icsdir: ${ICSDIR_ROOT}/C96C48 - idate: 2021122018 - edate: 2021122106 - nens: 2 - gfs_cyc: 1 - start: cold - yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml From 0b88e3284a5796ed8765ba37e59b54a5359550c4 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 00:48:40 +0000 Subject: [PATCH 135/150] Update globalworkflow-ci.yaml github.run_id not github.run.id !! --- .github/workflows/globalworkflow-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index c9582ac71c..5f21e15dbf 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -88,7 +88,7 @@ jobs: steps: - name: Create Experiments ${{ matrix.pslot }} env: - pslot: ${{ matrix.pslot }}_${{ github.run.id }} + pslot: ${{ matrix.pslot }}_${{ github.run_id }} run: | cd ${{ env.HOMEGFS }} source workflow/gw_setup.sh @@ -115,7 +115,7 @@ jobs: env: HOMEgfs: ${{ env.HOMEGFS }} EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR - pslot: ${{ matrix.pslot }}_${{ github.run.id }} + pslot: ${{ matrix.pslot }}_${{ github.run_id }} pr: ${{ github.event.pull_request.number }} run: | cd $HOMEgfs From e2068ba332472bb4bc0c98664f91ac23d7120d72 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 00:51:07 +0000 Subject: [PATCH 136/150] Update create_experiment.py still need pslot at end of arg to setup_xml --- ci/scripts/create_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index 1d79ca1882..67c5a73b16 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -99,7 +99,7 @@ def input_args(): setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr) setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py')) - expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir))) + expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir),Path(setup_expt_args.arguments.pslot))) setup_xml_cmd.add_default_arg(str(expdir)) logger.info(f'Run command: {setup_xml_cmd.command}') From cce01b2523f45be42ac4279aea3c4657cfe16742 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 12:26:46 +0000 Subject: [PATCH 137/150] fixed extra pslot removal in cron checks --- ci/scripts/check_ci.sh | 16 ++++++++-------- ci/scripts/run_ci.sh | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index 11ea7f29b5..7c4cefa894 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -77,11 +77,10 @@ for pr in ${pr_list}; do if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then continue fi - num_cases=$(find "${pr_dir}/RUNTESTS" -mindepth 1 -maxdepth 1 -type d | wc -l) || true - #Check for PR success when ${pr_dir}/RUNTESTS is void of subfolders + #Check for PR success when ${pr_dir}/RUNTESTS/EXPDIR is void of subfolders # since all successfull ones where previously removed - if [[ "${num_cases}" -eq 0 ]] && [[ -d "${pr_dir}/RUNTESTS" ]]; then + if [[ -z "$(ls -A ${pr_dir}/RUNTESTS/EXPDIR)" ]] ; then "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed" sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}" @@ -91,14 +90,14 @@ for pr in ${pr_list}; do continue fi - for cases in "${pr_dir}/RUNTESTS/"*; do + for cases in "${pr_dir}/RUNTESTS/EXPDIR/"*; do pslot=$(basename "${cases}") || true if [[ -z "${pslot+x}" ]]; then - echo "No cases found in ${pr_dir}/RUNTESTS .. exiting" + echo "No cases found in ${pr_dir}/RUNTESTS/EXPDIR .. exiting" exit 0 fi - xml="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.xml" - db="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.db" + xml="${pr_dir}/RUNTESTS/EXPDIR/${pslot}/${pslot}.xml" + db="${pr_dir}/RUNTESTS/EXPDIR/${pslot}/${pslot}.db" if [[ ! -f "${db}" ]]; then continue fi @@ -137,7 +136,8 @@ for pr in ${pr_list}; do sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}" #Remove Experment cases that completed successfully - rm -Rf "${pr_dir}/RUNTESTS/${pslot}" + rm -Rf "${pr_dir}/RUNTESTS/EXPDIR/${pslot}" + rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}" fi done done diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh index 116c7f3a7f..2091765fb3 100755 --- a/ci/scripts/run_ci.sh +++ b/ci/scripts/run_ci.sh @@ -79,8 +79,8 @@ for pr in ${pr_list}; do continue fi pslot=$(basename "${cases}") - xml="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.xml" - db="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.db" + xml="${pr_dir}/RUNTESTS/EXPDIR/${pslot}/${pslot}.xml" + db="${pr_dir}/RUNTESTS/EXPDIR/${pslot}/${pslot}.db" echo "Running: ${rocotorun} -v 6 -w ${xml} -d ${db}" "${rocotorun}" -v 10 -w "${xml}" -d "${db}" done From f5b68e1510800020d43af6ba9fa58845d13b8056 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 13:49:38 +0000 Subject: [PATCH 138/150] Update globalworkflow-ci.yaml old extra pslot still in EXPDIR path now removed --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 5f21e15dbf..d0e576c86c 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -114,7 +114,7 @@ jobs: - name: Run Experiment ${{ matrix.pslot }} env: HOMEgfs: ${{ env.HOMEGFS }} - EXPDIR: ${{ env.RUNTESTS }}/${{ matrix.pslot }}/EXPDIR + EXPDIR: ${{ env.RUNTESTS }}/EXPDIR pslot: ${{ matrix.pslot }}_${{ github.run_id }} pr: ${{ github.event.pull_request.number }} run: | From f293ee245b2ef41a771ec694a509ac5387637625 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 14:24:14 +0000 Subject: [PATCH 139/150] added max paralel exp run to two --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 5f21e15dbf..3df8ef87af 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -108,6 +108,7 @@ jobs: needs: create-experiments runs-on: [self-hosted, orion-ready] strategy: + max-parallel: 2 matrix: pslot: ["C48_S2S","C96_atm3DVar"] steps: From de12e3ed8fcf615d22d888eb07af4fd26588d944 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 12 Jul 2023 11:04:17 -0500 Subject: [PATCH 140/150] removed -u from checkout --- ci/scripts/clone-build_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index d06d4d732e..982655fd70 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -80,7 +80,7 @@ echo "${commit}" > "../commit" # run checkout script cd sorc || exit 1 set +e -./checkout.sh -c -g -u >> log.checkout 2>&1 +./checkout.sh -c -g >> log.checkout 2>&1 checkout_status=$? if [[ ${checkout_status} != 0 ]]; then { From b18dba007ae2477ccad8d0b40e429913297c3ffe Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 21:45:11 +0000 Subject: [PATCH 141/150] remove u from clone-build --- ci/scripts/clone-build_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index d06d4d732e..982655fd70 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -80,7 +80,7 @@ echo "${commit}" > "../commit" # run checkout script cd sorc || exit 1 set +e -./checkout.sh -c -g -u >> log.checkout 2>&1 +./checkout.sh -c -g >> log.checkout 2>&1 checkout_status=$? if [[ ${checkout_status} != 0 ]]; then { From 945ea5bbfca70c73796c209ed9cd1167194c4d76 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 22:07:59 +0000 Subject: [PATCH 142/150] Update globalworkflow-ci.yaml took out condition on label --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 6c13672e6a..6d6b9716ff 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -12,7 +12,7 @@ env: jobs: checkout-build-link: - if: github.event.label.name == 'CI-Orion-Ready' + # if: github.event.label.name == 'CI-Orion-Ready' runs-on: [self-hosted, orion-ready] timeout-minutes: 600 From acc70d53635faa7f02d838a074277873d1fa1abd Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 18:22:48 -0400 Subject: [PATCH 143/150] Update globalworkflow-ci.yaml one uses per set --- .github/workflows/globalworkflow-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 6d6b9716ff..7daf278563 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -93,6 +93,7 @@ jobs: uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Running" + - if: failure() uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" From 6e5d1bf8f4b2b5fefa3a5913392c47ebc1b6ba47 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 18:25:34 -0400 Subject: [PATCH 144/150] Update globalworkflow-ci.yaml two more only one uses per step --- .github/workflows/globalworkflow-ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 6d6b9716ff..40dc31f60c 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -93,6 +93,7 @@ jobs: uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Running" + - if: failure() uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" @@ -116,9 +117,11 @@ jobs: uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Running" + - if: failure() uses: actions-ecosystem/action-add-labels@v1 with: labels: "CI-Orion-Failed" + - if: failure() uses: actions-ecosystem/action-create-comment@v1 with: github_token: ${{ secrets.github_token }} From b34db89b3f7bc61d4bb6ebb34d010ef4de9be72e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 22:36:04 +0000 Subject: [PATCH 145/150] Update globalworkflow-ci.yaml or is || --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 40dc31f60c..cfef00384f 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -47,7 +47,7 @@ jobs: cd ${{ env.HOMEgfs_PR }}/sorc ./build_all.sh - - if: success() or failure() + - if: success() || failure() uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Building" From d7d2ef74ae60eade779a6c019de89261892c2389 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 18:38:53 -0400 Subject: [PATCH 146/150] Update globalworkflow-ci.yaml or is || --- .github/workflows/globalworkflow-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index cfef00384f..f69a6d267e 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -27,7 +27,7 @@ jobs: cd ${{ env.HOMEgfs_PR }}/sorc ./checkout.sh # Options e.g. -g -u can be added late - - if: success() or failure() + - if: success() || failure() uses: actions-ecosystem/action-remove-labels@v1 with: labels: "CI-Orion-Ready" From 0ac6230b842663c4692f02de8ad96ceed62f20a7 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 12 Jul 2023 23:03:00 +0000 Subject: [PATCH 147/150] replaced PR with run_id because dispatches are not on PRs --- .github/workflows/globalworkflow-ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 7daf278563..f1a3de4552 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -5,9 +5,9 @@ name: gw-ci-orion on: [workflow_dispatch] env: - pr: ${{ github.event.pull_request.number }} - HOMEgfs_PR: ${{ github.workspace }}/${{ github.event.pull_request.number }} - RUNTESTS: ${{ github.workspace }}/RUNTESTS/${{ github.event.pull_request.number }} + pr: ${{ github.run_id }} + HOMEgfs_PR: ${{ github.workspace }}/${{ github.run_id }} + RUNTESTS: ${{ github.workspace }}/RUNTESTS/${{ github.run_id }} MACHINE_ID: orion jobs: @@ -20,7 +20,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - path: ${{ github.event.pull_request.number }} + path: ${{ github.run_id }} - name: Checkout components run: | From 56241450f73504cee8c550d1e0e319784d8d77cc Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 12 Jul 2023 18:24:34 -0500 Subject: [PATCH 148/150] removed lables for now for dispatch version --- .github/workflows/globalworkflow-ci.yaml | 60 ------------------------ 1 file changed, 60 deletions(-) diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index bdcd729cac..338c4c025f 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -12,7 +12,6 @@ env: jobs: checkout-build-link: - # if: github.event.label.name == 'CI-Orion-Ready' runs-on: [self-hosted, orion-ready] timeout-minutes: 600 @@ -27,51 +26,16 @@ jobs: cd ${{ env.HOMEgfs_PR }}/sorc ./checkout.sh # Options e.g. -g -u can be added late - - if: success() || failure() - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: "CI-Orion-Ready" - - - if: success() - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Building" - - - if: failure() - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Failed" - - name: Build components run: | cd ${{ env.HOMEgfs_PR }}/sorc ./build_all.sh - - if: success() || failure() - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: "CI-Orion-Building" - - - if: failure() - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Failed" - - name: Link artifacts run: | cd ${{ env.HOMEgfs_PR }}/sorc ./link_workflow.sh - - if: failure() - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Failed" - - - if: success() - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Running" - create-experiments: needs: checkout-build-link runs-on: [self-hosted, orion-ready] @@ -89,15 +53,6 @@ jobs: source ci/platforms/orion.sh ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }} - - if: failure() - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: "CI-Orion-Running" - - if: failure() - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Failed" - run-experiments: needs: create-experiments runs-on: [self-hosted, orion-ready] @@ -113,21 +68,6 @@ jobs: cd ${{ env.HOMEgfs_PR }} ./ci/scripts/run-check_ci.sh ${{ env.HOMEgfs_PR }} ${{ env.RUNTESTS }}/EXPDIR ${{ matrix.case }}.${{ github.run_id }} - - if: failure() - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: "CI-Orion-Running" - - if: failure() - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Failed" - - if: failure() - uses: actions-ecosystem/action-create-comment@v1 - with: - github_token: ${{ secrets.github_token }} - body: | - ${{ env.HOMEgfs}}/global-workflow.log - release-lock: needs: run-experiments runs-on: [self-hosted, orion-ready] From 3656a48ea145287035643f6cc136e09957160757 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Wed, 12 Jul 2023 19:03:32 -0500 Subject: [PATCH 149/150] removed Rocoto utilities as vars as this scripts is not run in cron and does not need full paths --- ci/scripts/run-check_ci.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 1cacb278ef..1e1c47d49a 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -26,8 +26,8 @@ fi rc=99 while true; do - echo "Running: ${rocotorun} -v 10 -w ${xml} -d ${db}" - "${rocotorun}" -v 10 -w "${xml}" -d "${db}" + echo "Running: rocotorun -v 10 -w ${xml} -d ${db}" + rocotorun -v 10 -w "${xml}" -d "${db}" # Wait before running rocotostat sleep 30 @@ -37,11 +37,11 @@ while true; do fi # Get job statistics - rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true - num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true - num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true - num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true - num_failed=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true + rocoto_stat_output=$(rocotostat -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true + num_cycles=$(echo rocoto_stat_output | wc -l) || true + num_done=$(echo rocoto_stat_output | grep -c Done) || true + num_succeeded=$(rocotostat -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true + num_failed=$(rocotostat -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" @@ -51,7 +51,7 @@ while true; do echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true } >> "${HOMEgfs}/global-workflow.log" - error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true + error_logs=$(rocotostat -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs rocotocheck -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true { echo "Error logs:" echo "${error_logs}" From 8562d0ab8a5b93181d97ff94a155e3b411cab9bd Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 10 Aug 2023 11:53:39 -0500 Subject: [PATCH 150/150] added BASH wrapper to create_experiment.py for user access from command line, removed redundent seceond argument to it as well --- .github/workflows/globalworkflow-ci.yaml | 62 +++++++++-------- ci/scripts/create_experiment.py | 86 +++++++++--------------- ci/scripts/create_experiment.sh | 71 +++++++++++++++++++ ci/scripts/driver.sh | 7 +- 4 files changed, 141 insertions(+), 85 deletions(-) create mode 100755 ci/scripts/create_experiment.sh diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/globalworkflow-ci.yaml index 338c4c025f..e93d6259d0 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/globalworkflow-ci.yaml @@ -1,13 +1,20 @@ name: gw-ci-orion -# A GitHub Action that creates a deployment for a PR, clones, builds and runs a test suite -# This workflow is triggered by a label being added to a PR name Orion-Ready on: [workflow_dispatch] +# TEST_DIR contains 2 directories; +# 1. HOMEgfs: clone of the global-workflow +# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments +# e.g. $> tree ./TEST_DIR +# ./TEST_DIR +# ├── HOMEgfs +# └── RUNTESTS +# ├── COMROT +# │   └── ${pslot} +# └── EXPDIR +# └── ${pslot} env: - pr: ${{ github.run_id }} - HOMEgfs_PR: ${{ github.workspace }}/${{ github.run_id }} - RUNTESTS: ${{ github.workspace }}/RUNTESTS/${{ github.run_id }} + TEST_DIR: ${{ github.workspace }}/${{ github.run_id }} MACHINE_ID: orion jobs: @@ -16,24 +23,24 @@ jobs: timeout-minutes: 600 steps: - - name: Checkout + - name: Checkout global-workflow uses: actions/checkout@v3 with: - path: ${{ github.run_id }} + path: ${{ github.run_id }}/HOMEgfs # This path needs to be relative - name: Checkout components run: | - cd ${{ env.HOMEgfs_PR }}/sorc - ./checkout.sh # Options e.g. -g -u can be added late + cd ${{ env.TEST_DIR }}/HOMEgfs/sorc + ./checkout.sh -c -g # Options e.g. -u can be added late - name: Build components run: | - cd ${{ env.HOMEgfs_PR }}/sorc + cd ${{ env.TEST_DIR }}/HOMEgfs/sorc ./build_all.sh - name: Link artifacts run: | - cd ${{ env.HOMEgfs_PR }}/sorc + cd ${{ env.TEST_DIR }}/HOMEgfs/sorc ./link_workflow.sh create-experiments: @@ -41,17 +48,19 @@ jobs: runs-on: [self-hosted, orion-ready] strategy: matrix: - case: ["C48_S2S","C96_atm3DVar"] + case: ["C48_S2S", "C96_atm3DVar"] steps: - name: Create Experiments ${{ matrix.case }} env: + HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs + RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS pslot: ${{ matrix.case }}.${{ github.run_id }} run: | - cd ${{ env.HOMEgfs_PR }} - source workflow/gw_setup.sh - source ci/platforms/orion.sh - ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }} + cd ${{ env.TEST_DIR }}/HOMEgfs + source workflow/gw_setup.sh + source ci/platforms/orion.sh + ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml run-experiments: needs: create-experiments @@ -59,22 +68,19 @@ jobs: strategy: max-parallel: 2 matrix: - case: ["C48_S2S", "C96_atm3DVar"] + case: ["C48_S2S", "C96_atm3DVar"] steps: - name: Run Experiment ${{ matrix.case }} - env: - pr: ${{ env.pr }} run: | - cd ${{ env.HOMEgfs_PR }} - ./ci/scripts/run-check_ci.sh ${{ env.HOMEgfs_PR }} ${{ env.RUNTESTS }}/EXPDIR ${{ matrix.case }}.${{ github.run_id }} + cd ${{ env.TEST_DIR }}/HOMEgfs + ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} - release-lock: + clean-up: needs: run-experiments runs-on: [self-hosted, orion-ready] steps: - - uses: actions-ecosystem/action-remove-labels@v1 - with: - labels: "CI-Orion-Running" - - uses: actions-ecosystem/action-add-labels@v1 - with: - labels: "CI-Orion-Passed" + - name: Clean-up + run: | + cd ${{ github.workspace }} + rm -rf ${{ github.run_id }} + diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index 67c5a73b16..cb96b5f1c0 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -2,107 +2,87 @@ """ Basic python script to create an experiment directory on the fly from a given - yaml file for the arguments to the two scripts below in ${HOMEgfs}/workflow - where ${HOMEgfs} is specified within the input yaml file. ${HOMEgfs}/workflow/setup_expt.py ${HOMEgfs}/workflow/setup_xml.py The yaml file are simply the arguments for these two scripts. -After this scripts runs these two the use will have an experiment ready for launching +After this scripts runs the experiment is ready for launch. Output ------ - Functionally an experiment is setup as a result running the two scripts described above with an error code of 0 upon success. """ import os +import sys from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from pathlib import Path -from wxflow import YAMLFile, Logger, logit, Executable +from wxflow import YAMLFile, Logger, logit +_here = os.path.dirname(__file__) +_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../..')) logger = Logger(level='DEBUG', colored_log=True) +# TODO: move create_experiment.py to workflow/ and remove this sys.path.insert business +sys.path.insert(0, os.path.join(_top, 'workflow')) +import setup_expt +import setup_xml + + @logit(logger) def input_args(): """ - Method to collect user arguments for `create_experiment.py` - - Input - ----- - - A single key valued argument: --yaml - Description ----------- - A full path to a YAML file with the following format with required sections: experiment, arguments + Method to collect user arguments for `create_experiment.py` - experiment: - mode: - used to hold the only required positional argument to setup_expt.py + Parameters + ---------- - arguments: - holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py - Note: the argument pslot is derived from the basename of the yamlfile itself + None Returns ------- - args: Namespace - - Namespace with the value of the file path to a yaml file from the key yaml + argparse.Namespace: + argparse.Namespace with the value of the file path to a yaml file from the key yaml """ - description = """Single argument as a yaml file containing the - key value pairs as arguments to setup_expt.py - """ + description = """Create a global-workflow experiment""" parser = ArgumentParser(description=description, formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--yaml', help='yaml configuration file per experiment', type=str, required=True) - parser.add_argument('--dir', help='full path to top level of repo of global-workflow', type=str, required=True) + parser.add_argument('--yaml', help='full path to yaml file describing the experiment configuration', type=str, required=True) - args = parser.parse_args() - return args + return parser.parse_args() if __name__ == '__main__': user_inputs = input_args() - setup_expt_args = YAMLFile(path=user_inputs.yaml) - - HOMEgfs = Path.absolute(Path(user_inputs.dir)) - type = setup_expt_args.experiment.type - mode = setup_expt_args.experiment.mode - - setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py')) - - setup_expt_cmd.add_default_arg(type) - setup_expt_cmd.add_default_arg(mode) + testconf = YAMLFile(path=user_inputs.yaml) + experiment_dir = Path.absolute(Path.joinpath(Path(testconf.arguments.expdir), Path(testconf.arguments.pslot))) - for conf, value in setup_expt_args.arguments.items(): - setup_expt_cmd.add_default_arg(f'--{conf}') - setup_expt_cmd.add_default_arg(str(value)) + # Create a list of arguments to setup_expt.py + setup_expt_args = [testconf.experiment.type, testconf.experiment.mode] # TODO: rename 'type' as 'system' in case.yaml + for kk, vv in testconf.arguments.items(): + setup_expt_args.append(f"--{kk}") + setup_expt_args.append(str(vv)) - logger.info(f'Run command: {setup_expt_cmd.command}') - setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr')) - setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout')) - setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr) + logger.info(f'Call: setup_expt.main()') + setup_expt.main(setup_expt_args) - setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py')) - expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir),Path(setup_expt_args.arguments.pslot))) - setup_xml_cmd.add_default_arg(str(expdir)) + # Create a list of arguments to setup_xml.py + setup_xml_args = [str(experiment_dir)] - logger.info(f'Run command: {setup_xml_cmd.command}') - setup_xml_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stderr')) - setup_xml_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stdout')) - setup_xml_cmd(output=setup_xml_stdout, error=setup_xml_stderr) + logger.info(f"Call: setup_xml.main()") + setup_xml.main(setup_xml_args) diff --git a/ci/scripts/create_experiment.sh b/ci/scripts/create_experiment.sh new file mode 100755 index 0000000000..7aa68b3712 --- /dev/null +++ b/ci/scripts/create_experiment.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -eux + +######################################################################### +# +# Script description: BASH script for creating an experiment +# by only taking a single YAML file +######################################################################## + +usage() { + set +x + echo + echo "Usage: $0 -y case/yaml_conf_file.yaml" + echo + echo " -c global workflow configuration yaml defining an experiment (\$HOMEgfs/ci/cases)" + echo " -d full path to directory to place COMROT with EXPDIR for createing experiments" + echo " -h display this message and quit" + echo + exit 1 +} + +while getopts "c:d:h" opt; do + case ${opt} in + c) + YAML_CASE=${OPTARG} + ;; + d) + RUNTESTS=${OPTARG} + ;; + h|\?|:) + usage + ;; + *) + echo "Unrecognized option" + usage + ;; + esac +done + +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" +scriptname=$(basename "${BASH_SOURCE[0]}") +echo "Begin ${scriptname} at $(date -u)" || true +export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' + +######################################################################### +# Set up runtime environment varibles for accounts on supproted machines +######################################################################### + +source "${HOMEgfs}/ush/detect_machine.sh" +case ${MACHINE_ID} in + hera | orion) + echo "Running Automated Testing on ${MACHINE_ID}" + source "${HOMEgfs}/ci/platforms/${MACHINE_ID}.sh" + ;; + *) + echo "Unsupported platform. Exiting with error." + exit 1 + ;; +esac + +source "${HOMEgfs}/ci/platforms/${MACHINE_ID}.sh" + +filename=$(basename -- ${YAML_CASE}) +export pslot="${filename%.*}" +export RUNTESTS=${RUNTESTS:-"${PWD}"} +# TODO env HOMEgfs being set will cause runtime error +# with HOMEgfs being blank in Rocoto XLM file +export HOMEgfs_PR="${HOMEgfs}" +unset HOMEgfs; + +${HOMEgfs_PR}/ci/scripts/create_experiment.py --yaml $YAML_CASE diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index 728b0d5cd9..cdc87129c1 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -13,7 +13,6 @@ set -eux # PR number and calls clone-build_ci.sh to perform a clone and full build from $(HOMEgfs)/sorc # of the PR. It then is ready to run a suite of regression tests with various # configurations with run_tests.py. -# No-op for test ####################################################################################### ################################################################# @@ -149,14 +148,14 @@ for pr in ${pr_list}; do HOMEgfs_PR="${pr_dir}/global-workflow" export HOMEgfs_PR cd "${HOMEgfs_PR}" - discribe=$(git rev-parse --short HEAD) + pr_sha=$(git rev-parse --short HEAD) for yaml_config in "${HOMEgfs_PR}/ci/cases/"*.yaml; do case=$(basename "${yaml_config}" .yaml) || true - pslot="${pslot}_${discribe}" + pslot="${case}_${pr_sha}" export pslot set +e - "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml" --dir "${HOMEgfs_PR}" + "${HOMEgfs_PR}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml" ci_status=$? set -e if [[ ${ci_status} -eq 0 ]]; then