From ac3cde510d4aeed632e3b080b6227cb5a26ef6d0 Mon Sep 17 00:00:00 2001 From: BoCui-NOAA <53531984+BoCui-NOAA@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:54:28 -0400 Subject: [PATCH 01/21] Update memory settings in config.resources for job postsnd (#3003) # Description This PR corrects the memory settings for the BUFR sounding job postsnd. The "memory" specified in the config.resources file represents the total memory required for the entire job, not for a single processor. The updated config.resources file removes the "memory" entries for C1152 and C768 for the postsnd job. # Type of change - [x] Bug fix (fixes something broken) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? - Cycled test on WCOSS2 # Checklist - [ ] Any dependent changes have been merged and published - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have documented my code, including function, input, and output descriptions - [ ] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added - [ ] I have made corresponding changes to the system documentation if necessary --------- Co-authored-by: Walter Kolczynski - NOAA --- parm/config/gfs/config.resources | 2 -- 1 file changed, 2 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 5024bc8873..26f6126773 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -1214,12 +1214,10 @@ case ${step} in "C768") tasks_per_node=21 threads_per_task=6 - memory="23GB" ;; "C1152") tasks_per_node=9 threads_per_task=14 - memory="50GB" ;; *) tasks_per_node=21 From e1fcce1465732f22fc9b6a89323f9923b73264cb Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 15 Oct 2024 16:40:26 -0400 Subject: [PATCH 02/21] Use ubuntu-22.04 on github runner (#3005) This PR reverts to `ubuntu-22.04` image of the GH runner. `ubuntu-latest` is having an issue with python pip. --- .github/workflows/ci_unit_tests.yaml | 4 ++-- .github/workflows/docs.yaml | 2 +- .github/workflows/hera.yaml | 2 +- .github/workflows/linters.yaml | 2 +- .github/workflows/orion.yaml | 4 ++-- .github/workflows/pynorms.yaml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_unit_tests.yaml b/.github/workflows/ci_unit_tests.yaml index 6dbc7ee52c..5fa2ec28c1 100644 --- a/.github/workflows/ci_unit_tests.yaml +++ b/.github/workflows/ci_unit_tests.yaml @@ -4,8 +4,8 @@ on: [pull_request, push, workflow_dispatch] jobs: ci_pytest: - runs-on: ubuntu-latest - name: Run unit tests on CI system + runs-on: ubuntu-22.04 + name: Run unit tests on CI system permissions: checks: write diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b04c85688b..89b5fb617a 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -21,7 +21,7 @@ jobs: permissions: pull-requests: 'write' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 name: Build and deploy documentation steps: diff --git a/.github/workflows/hera.yaml b/.github/workflows/hera.yaml index 800d87e55a..0dd9f2c356 100644 --- a/.github/workflows/hera.yaml +++ b/.github/workflows/hera.yaml @@ -9,7 +9,7 @@ on: jobs: getlabels: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: labels: ${{ steps.id.outputs.labels }} steps: diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index 488b6a1407..7816788b81 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -12,7 +12,7 @@ defaults: jobs: lint-shell: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: security-events: write diff --git a/.github/workflows/orion.yaml b/.github/workflows/orion.yaml index 2d17b3db63..aaf1e28370 100644 --- a/.github/workflows/orion.yaml +++ b/.github/workflows/orion.yaml @@ -9,7 +9,7 @@ on: jobs: getlabels: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: labels: ${{ steps.id.outputs.labels }} steps: @@ -27,7 +27,7 @@ jobs: passed: if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') && github.event.pull_request.merged - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: - getlabels diff --git a/.github/workflows/pynorms.yaml b/.github/workflows/pynorms.yaml index 6ea99b59ed..87915190ef 100644 --- a/.github/workflows/pynorms.yaml +++ b/.github/workflows/pynorms.yaml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: check_norms: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 name: Check Python coding norms with pycodestyle steps: From ca8d9f4a8dcb1bdb7be066866e96321cc5c1bf86 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 16 Oct 2024 15:51:11 -0400 Subject: [PATCH 03/21] Github pipelines and utils for running CI on parallel works (#3007) # Description This PR has the GitHub Pipeline script in the `github/workflows` directory for running CI tests to be preformed an AWS virtual cluster. It is setup to be launched from the dispatch action from the Actions tab. For now it will only run C48_ATM Resolves #3006 Once the yaml pipeline is in `.github/workflows` directory of the default branch we can test it against [PR 2977](https://github.com/NOAA-EMC/global-workflow/pull/2977) which may be needed to build on Parallel Works Centos AWS. Code managers can check to see if the self-hosted runner [globalworkflow_parallelworks](https://github.com/NOAA-EMC/global-workflow/settings/actions/runners/22) is up and ready by checking the [Running](https://github.com/NOAA-EMC/global-workflow/settings/actions/runners) Settings. In pending work we should also be able spin up the cluster on demand from GitHub as well. # Type of change - [ ] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [x] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? YES/NO - Does this change require a documentation update? YES/NO - Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.) - [ ] EMC verif-global - [ ] GDAS - [ ] GFS-utils - [ ] GSI - [ ] GSI-monitor - [ ] GSI-utils - [ ] UFS-utils - [ ] UFS-weather-model - [ ] wxflow # How has this been tested? # Checklist - [ ] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have documented my code, including function, input, and output descriptions - [ ] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [x] This change is covered by an existing CI test or a new one has been added - [ ] I have made corresponding changes to the system documentation if necessary --------- Co-authored-by: tmcguinness --- ...balworkflow-ci.yaml => pw_aws_centos.yaml} | 46 ++++++++++--------- .../parallel_works/UserBootstrap_centos7.txt | 5 ++ .../utils/parallel_works/provision_runner.sh | 39 ++++++++++++++++ 3 files changed, 68 insertions(+), 22 deletions(-) rename .github/workflows/{globalworkflow-ci.yaml => pw_aws_centos.yaml} (63%) create mode 100644 ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt create mode 100755 ci/scripts/utils/parallel_works/provision_runner.sh diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/pw_aws_centos.yaml similarity index 63% rename from .github/workflows/globalworkflow-ci.yaml rename to .github/workflows/pw_aws_centos.yaml index 1474c79a48..549a3ea0fa 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/pw_aws_centos.yaml @@ -1,4 +1,4 @@ -name: gw-ci-orion +name: gw-ci-aws-centos on: [workflow_dispatch] @@ -15,28 +15,31 @@ on: [workflow_dispatch] # └── ${pslot} env: TEST_DIR: ${{ github.workspace }}/${{ github.run_id }} - MACHINE_ID: orion + MACHINE_ID: noaacloud jobs: - checkout-build-link: - runs-on: [self-hosted, orion-ready] + checkout: + runs-on: [self-hosted, aws, parallelworks, centos] timeout-minutes: 600 steps: + - name: Checkout global-workflow - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - path: ${{ github.run_id }}/HOMEgfs # This path needs to be relative + path: ${{ github.run_id }}/HOMEgfs + submodules: 'recursive' - - name: Checkout components - run: | - cd ${{ env.TEST_DIR }}/HOMEgfs/sorc - ./checkout.sh -c -g # Options e.g. -u can be added late + build-link: + runs-on: [self-hosted, aws, parallelworks, centos] + needs: checkout + + steps: - name: Build components run: | cd ${{ env.TEST_DIR }}/HOMEgfs/sorc - ./build_all.sh + ./build_all.sh -j 8 - name: Link artifacts run: | @@ -44,43 +47,42 @@ jobs: ./link_workflow.sh create-experiments: - needs: checkout-build-link - runs-on: [self-hosted, orion-ready] + needs: checkout + runs-on: [self-hosted, aws, parallelworks, centos] strategy: matrix: - case: ["C48_S2S", "C96_atm3DVar"] + case: ["C48_ATM"] steps: - name: Create Experiments ${{ matrix.case }} env: - HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS pslot: ${{ matrix.case }}.${{ github.run_id }} run: | + mkdir -p ${{ env.RUNTESTS }} cd ${{ env.TEST_DIR }}/HOMEgfs source workflow/gw_setup.sh - source ci/platforms/orion.sh - ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }} + source ci/platforms/config.noaacloud + ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite run-experiments: needs: create-experiments - runs-on: [self-hosted, orion-ready] + runs-on: [self-hosted, aws, parallelworks, centos] strategy: max-parallel: 2 matrix: - case: ["C48_S2S", "C96_atm3DVar"] + case: ["C48_ATM"] steps: - name: Run Experiment ${{ matrix.case }} run: | cd ${{ env.TEST_DIR }}/HOMEgfs - ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} + ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs clean-up: needs: run-experiments - runs-on: [self-hosted, orion-ready] + runs-on: [self-hosted, aws, parallelworks, centos] steps: - name: Clean-up run: | cd ${{ github.workspace }} rm -rf ${{ github.run_id }} - diff --git a/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt b/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt new file mode 100644 index 0000000000..ddc6b05706 --- /dev/null +++ b/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt @@ -0,0 +1,5 @@ +sudo yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm +sudo yum -y install git +/contrib/Terry.McGuinness/SETUP/provision_runner.sh +ALLNODES +/contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh \ No newline at end of file diff --git a/ci/scripts/utils/parallel_works/provision_runner.sh b/ci/scripts/utils/parallel_works/provision_runner.sh new file mode 100755 index 0000000000..cac18c9315 --- /dev/null +++ b/ci/scripts/utils/parallel_works/provision_runner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# This script provisions a GitHub Actions runner on a Rocky or CentOS system. +# It performs the following steps: +# 1. Checks the operating system from /etc/os-release. +# 2. Verifies if the operating system is either Rocky or CentOS. +# 3. Checks if an actions-runner process is already running for the current user. +# 4. Copies the actions-runner tar file from a specified directory to the home directory. +# 5. Extracts the tar file and starts the actions-runner in the background. +# +# The actions-runner tar file contains the necessary binaries and scripts to run +# a GitHub Actions runner. It is specific to the operating system and is expected +# to be located in the /contrib/${CI_USER}/SETUP/ directory. + +CI_USER="Terry.McGuinness" + +# Get the Operating System name from /etc/os-release +OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/') || true + +# Check if the OS is Rocky or CentOS +if [[ "${OS_NAME}" == "rocky" || "${OS_NAME}" == "centos" ]]; then + echo "Operating System is ${OS_NAME}" +else + echo "Unsupported Operating System: ${OS_NAME}" + exit 1 +fi + +running=$(pgrep -u "${USER}" run-helper -c) || true +if [[ "${running}" -gt 0 ]]; then + echo "actions-runner is already running" + exit +fi + +cp "/contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz" "${HOME}" +cd "${HOME}" || exit +tar -xf "actions-runner_${OS_NAME}.tar.gz" +cd actions-runner || exit +d=$(date +%Y-%m-%d-%H:%M) +nohup ./run.sh >& "run_nohup${d}.log" & From 77edb05e3fea4003e7db607bef3712d9fdc932be Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 16 Oct 2024 16:14:43 -0400 Subject: [PATCH 04/21] Added ref to checkout for pull request on GitHub dispatch (#3010) # Description The checkout reference for the GitHub dispatch action needed to be update to specify PRs. # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) Co-authored-by: Terry McGuinness --- .github/workflows/pw_aws_centos.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pw_aws_centos.yaml b/.github/workflows/pw_aws_centos.yaml index 549a3ea0fa..76258c3044 100644 --- a/.github/workflows/pw_aws_centos.yaml +++ b/.github/workflows/pw_aws_centos.yaml @@ -29,6 +29,7 @@ jobs: with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' + ref: ${{ github.event.pull_request.head.ref }} build-link: runs-on: [self-hosted, aws, parallelworks, centos] From 82710f47c8d88b92a48f84e51e12f9d4a0680c75 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 17 Oct 2024 16:04:30 -0400 Subject: [PATCH 05/21] Add CODEOWNERS (#2996) This PR: - adds a `CODEOWNERS` file to the repository. global-workflow is a collaborative space where contributions come from a variety of sources. This file will ensure that new development gets reviewed by the appropriate SME. --- .github/CODEOWNERS | 210 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..e3521739c2 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,210 @@ +# global-workflow is a collaborative space where contributions come from a variety of sources +# This file is to ensure that new development gets reviewed by the appropriate SME + +# global-workflow default owners (not a complete list) +@KateFriedman-NOAA +@WalterKolczynski-NOAA +@DavidHuber-NOAA + +# Specific directory owners +/ci/ @TerrenceMcGuinness-NOAA @WalterKolczynski-NOAA + +/ecf/ @lgannoaa + +/workflow/ @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA + +# Specific file owners +# build scripts +sorc/build_*.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA +sorc/link_workflow.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA + +# jobs +jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @CoryMartin-NOAA +jobs/JGDAS_ATMOS_ANALYSIS_DIAG @RussTreadon-NOAA @CoryMartin-NOAA +jobs/JGDAS_ATMOS_CHGRES_FORENKF @RussTreadon-NOAA @CoryMartin-NOAA +jobs/JGDAS_ATMOS_GEMPAK @GwenChen-NOAA +jobs/JGDAS_ATMOS_GEMPAK_META_NCDC @GwenChen-NOAA +jobs/JGDAS_ATMOS_VERFOZN @EdwardSafford-NOAA +jobs/JGDAS_ATMOS_VERFRAD @EdwardSafford-NOAA +jobs/JGDAS_ENKF_* @RussTreadon-NOAA @CoryMartin-NOAA @CatherineThomas-NOAA +jobs/JGDAS_FIT2OBS @jack-woollen +jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN @guillaumevernieres +jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY @guillaumevernieres +jobs/JGFS_ATMOS_AWIPS_20KM_1P0DEG @GwenChen-NOAA +jobs/JGFS_ATMOS_CYCLONE_GENESIS @JiayiPeng-NOAA +jobs/JGFS_ATMOS_CYCLONE_TRACKER @JiayiPeng-NOAA +jobs/JGFS_ATMOS_FBWIND @GwenChen-NOAA +jobs/JGFS_ATMOS_FSU_GENESIS +jobs/JGFS_ATMOS_GEMPAK @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_META @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_NCDC_UPAPGIF @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC @GwenChen-NOAA +jobs/JGFS_ATMOS_PGRB2_SPEC_NPOESS @WenMeng-NOAA +jobs/JGFS_ATMOS_POSTSND @BoCui-NOAA +jobs/JGFS_ATMOS_VERIFICATION +jobs/JGLOBAL_AERO_ANALYSIS_* @CoryMartin-NOAA +jobs/JGLOBAL_ARCHIVE @DavidHuber-NOAA +jobs/JGLOBAL_ATMENS_ANALYSIS_* @RussTreadon-NOAA @CoryMartin-NOAA @DavidNew-NOAA +jobs/JGLOBAL_ATMOS_ANALYSIS @RussTreadon-NOAA @CatherineThomas-NOAA +jobs/JGLOBAL_ATMOS_ANALYSIS_CALC @RussTreadon-NOAA @CatherineThomas-NOAA @CoryMartin-NOAA +jobs/JGLOBAL_ATMOS_EMCSFC_SFC_PREP @GeorgeGayno-NOAA +jobs/JGLOBAL_ATMOS_ENSSTAT +jobs/JGLOBAL_ATMOS_POST_MANAGER +jobs/JGLOBAL_ATMOS_PRODUCTS @WenMeng-NOAA +jobs/JGLOBAL_ATMOS_SFCANL @GeorgeGayno-NOAA +jobs/JGLOBAL_ATMOS_TROPCY_QC_RELOC +jobs/JGLOBAL_ATMOS_UPP @WenMeng-NOAA +jobs/JGLOBAL_ATMOS_VMINMON @EdwardSafford-NOAA +jobs/JGLOBAL_ATM_* @RussTreadon-NOAA @DavidNew-NOAA @CoryMartin-NOAA +jobs/JGLOBAL_CLEANUP @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA +jobs/JGLOBAL_EXTRACTVARS @EricSinsky-NOAA +jobs/JGLOBAL_FORECAST @aerorahul +jobs/JGLOBAL_MARINE_* @guillaumevernieres @AndrewEichmann-NOAA +jobs/JGLOBAL_OCEANICE_PRODUCTS @GwenChen-NOAA +jobs/JGLOBAL_PREP_EMISSIONS @bbakernoaa +jobs/JGLOBAL_PREP_OBS_AERO @CoryMartin-NOAA +jobs/JGLOBAL_PREP_OCEAN_OBS @guillaumevernieres @AndrewEichmann-NOAA +jobs/JGLOBAL_*SNOW* @jiaruidong2017 +jobs/JGLOBAL_STAGE_IC @KateFriedman-NOAA +jobs/JGLOBAL_WAVE_* @JessicaMeixner-NOAA @sbanihash +jobs/rocoto/* @WalterKolczynski-NOAA @KateFriedman-NOAA @DavidHuber-NOAA + +# scripts +scripts/exgdas_aero_analysis_generate_bmatrix.py @CoryMartin-NOAA +scripts/exgdas_atmos_chgres_forenkf.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exgdas_atmos_gempak_gif_ncdc.sh @GwenChen-NOAA +scripts/exgdas_atmos_nawips.sh @GwenChen-NOAA +scripts/exgdas_atmos_verfozn.sh @EdwardSafford-NOAA +scripts/exgdas_atmos_verfrad.sh @EdwardSafford-NOAA +scripts/exgdas_enkf_earc.py @DavidHuber-NOAA +scripts/exgdas_enkf_ecen.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_post.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_select_obs.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_sfc.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_snow_recenter.py @jiaruidong2017 +scripts/exgdas_enkf_update.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_global_marine_analysis_letkf.py @guillaumevernieres @AndrewEichmann-NOAA +scripts/exgfs_aero_init_aerosol.py @WalterKolczynski-NOAA +scripts/exgfs_atmos_awips_20km_1p0deg.sh @GwenChen-NOAA +scripts/exgfs_atmos_fbwind.sh @GwenChen-NOAA +scripts/exgfs_atmos_gempak_gif_ncdc_skew_t.sh @GwenChen-NOAA +scripts/exgfs_atmos_gempak_meta.sh @GwenChen-NOAA +scripts/exgfs_atmos_goes_nawips.sh @GwenChen-NOAA +scripts/exgfs_atmos_grib2_special_npoess.sh @WenMeng-NOAA +scripts/exgfs_atmos_nawips.sh @GwenChen-NOAA +scripts/exgfs_atmos_postsnd.sh @BoCui-NOAA +scripts/exgfs_pmgr.sh +scripts/exgfs_prdgen_manager.sh +scripts/exgfs_wave_* @JessicaMeixner-NOAA @sbanihash +scripts/exglobal_aero_analysis_* @CoryMartin-NOAA +scripts/exglobal_archive.py @DavidHuber-NOAA +scripts/exglobal_atm_analysis_* @RussTreadon-NOAA @DavidNew-NOAA +scripts/exglobal_atmens_analysis_* @RussTreadon-NOAA @DavidNew-NOAA +scripts/exglobal_atmos_analysis*.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exglobal_atmos_ensstat.sh @RussTreadon-NOAA +scripts/exglobal_atmos_pmgr.sh +scripts/exglobal_atmos_products.sh @WenMeng-NOAA +scripts/exglobal_atmos_sfcanl.sh @GeorgeGayno-NOAA +scripts/exglobal_atmos_tropcy_qc_reloc.sh +scripts/exglobal_atmos_upp.py @WenMeng-NOAA +scripts/exglobal_atmos_vminmon.sh @EdwardSafford-NOAA +scripts/exglobal_cleanup.sh @DavidHuber-NOAA +scripts/exglobal_diag.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exglobal_extractvars.sh @EricSinsky-NOAA +scripts/exglobal_forecast.py @aerorahul +scripts/exglobal_forecast.sh @aerorahul @WalterKolczynski-NOAA +scripts/exglobal_marine_analysis_* @guillaumevernieres @AndrewEichmann-NOAA +scripts/exglobal_marinebmat.py @guillaumevernieres @AndrewEichmann-NOAA +scripts/exglobal_oceanice_products.py @GwenChen-NOAA +scripts/exglobal_prep_emissions.py @bbakernoaa +scripts/exglobal_prep_obs_aero.py @CoryMartin-NOAA +scripts/exglobal_prep_snow_obs.py @jiaruidong2017 +scripts/exglobal_snow_analysis.py @jiaruidong2017 +scripts/exglobal_stage_ic.py @KateFriedman-NOAA + +# ush +WAM_XML_to_ASCII.pl +atmos_ensstat.sh +atmos_extractvars.sh @EricSinsky-NOAA +bash_utils.sh @WalterKolczynski-NOAA +calcanl_gfs.py @CoryMartin-NOAA +calcinc_gfs.py @CoryMartin-NOAA +compare_f90nml.py @WalterKolczynski-NOAA @aerorahul +detect_machine.sh @WalterKolczynski-NOAA +extractvars_tools.sh @EricSinsky-NOAA +file_utils.sh @WalterKolczynski-NOAA +forecast_det.sh @aerorahul @WalterKolczynski-NOAA +forecast_postdet.sh @aerorahul @WalterKolczynski-NOAA +forecast_predet.sh @aerorahul @WalterKolczynski-NOAA +fv3gfs_remap_weights.sh +gaussian_sfcanl.sh @GeorgeGayno-NOAA +getdump.sh @WalterKolczynski-NOAA @KateFriedman-NOAA +getges.sh @WalterKolczynski-NOAA @KateFriedman-NOAA +getgfsnctime @CoryMartin-NOAA +getncdimlen @CoryMartin-NOAA +gfs_bfr2gpk.sh @GwenChen-NOAA +gfs_bufr.sh @GwenChen-NOAA +gfs_bufr_netcdf.sh @GwenChen-NOAA +gfs_sndp.sh @BoCui-NOAA +gfs_truncate_enkf.sh @CoryMartin-NOAA +global_savefits.sh +gsi_utils.py @CoryMartin-NOAA +interp_atmos_master.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA +interp_atmos_sflux.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA +jjob_header.sh @WalterKolczynski-NOAA +link_crtm_fix.sh @WalterKolczynski-NOAA +load_fv3gfs_modules.sh @WalterKolczynski-NOAA @aerorahul +load_ufsda_modules.sh @WalterKolczynski-NOAA @aerorahul @CoryMartin-NOAA +load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA +merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA +minmon_xtrct_*.pl @EdwardSafford-NOAA +module-setup.sh @WalterKolczynski-NOAA @aerorahul +oceanice_nc2grib2.sh @GwenChen-NOAA +ocnice_extractvars.sh @EricSinsky-NOAA +ozn_xtrct.sh @EdwardSafford-NOAA +parse-storm-type.pl +parsing_model_configure_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +parsing_namelists_CICE.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @DeniseWorthen +parsing_namelists_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +parsing_namelists_FV3_nest.sh @guoqing-noaa +parsing_namelists_MOM6.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @jiandewang +parsing_namelists_WW3.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA @sbanihash +parsing_ufs_configure.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +preamble.sh @WalterKolczynski-NOAA +product_functions.sh @WalterKolczynski-NOAA @aerorahul +radmon_*.sh @EdwardSafford-NOAA +rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA +run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA +syndat_getjtbul.sh @JiayiPeng-NOAA +syndat_qctropcy.sh @JiayiPeng-NOAA +tropcy_relocate.sh @JiayiPeng-NOAA +tropcy_relocate_extrkr.sh @JiayiPeng-NOAA +wave_*.sh @JessicaMeixner-NOAA @sbanihash + +# ush/python +ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA +ush/python/pygfs/jedi/jedi.py @DavidNew-NOAA +ush/python/pygfs/task/__init__.py @aerorahul +ush/python/pygfs/task/aero_analysis.py @DavidNew-NOAA @CoryMartin-NOAA +ush/python/pygfs/task/aero_bmatrix.py @DavidNew-NOAA @CoryMartin-NOAA +ush/python/pygfs/task/aero_emissions.py @bbakernoaa +ush/python/pygfs/task/aero_prepobs.py @CoryMartin-NOAA +ush/python/pygfs/task/analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/archive.py @DavidHuber-NOAA +ush/python/pygfs/task/atm_analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/atmens_analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/bmatrix.py @DavidNew-NOAA +ush/python/pygfs/task/gfs_forecast.py @aerorahul +ush/python/pygfs/task/marine_analysis.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/marine_bmat.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/marine_letkf.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/oceanice_products.py @aerorahul @GwenChen-NOAA +ush/python/pygfs/task/snow_analysis.py @jiaruidong2017 +ush/python/pygfs/task/snowens_analysis.py @jiaruidong2017 +ush/python/pygfs/task/stage_ic.py @KateFriedman-NOAA +ush/python/pygfs/task/upp.py @aerorahul @WenMeng-NOAA +ush/python/pygfs/ufswm/__init__.py @aerorahul +ush/python/pygfs/ufswm/gfs.py @aerorahul +ush/python/pygfs/ufswm/ufs.py @aerorahul +ush/python/pygfs/utils/__init__.py @aerorahul +ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA From ed3383da90e6a10df18642bffab633925a8a25ff Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Mon, 21 Oct 2024 06:29:30 -0400 Subject: [PATCH 06/21] Update PR template for code owners (#3020) # Description Adds a new checklist item for PRs that confirms any new scripts have been added to the CODEOWNERS file, along with owners. Follow-up to #2996 # Type of change - [x] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? N/A # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added - [x] I have made corresponding changes to the system documentation if necessary --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9a1d61eb30..9e9c9eccfe 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -72,4 +72,5 @@ Example: - [ ] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added +- [ ] Any new scripts have been added to the .github/CODEOWNERS file with owners - [ ] I have made corresponding changes to the system documentation if necessary From d9d30a1aecf76b4394b71d1cba6ad89913f705c0 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:40:54 -0400 Subject: [PATCH 07/21] Hotfix: Correctly set overwrite option when specified (#3021) # Description This specifies a correct option for the `overwrite` option in `setup_expt.py` when called with the `--overwrite` flag. As-is, the `EXPDIR` and `COMROOT` directories are not actually deleted when `--overwrite` is specified. This also removes an unused module (`warnings`) and method (`to_timedelta`) from setup_expt.py. # Type of change - [x] Bug fix (fixes something broken) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? Ran setup_expt.py with `--overwrite` and verified the `EXPDIR` and `COMROOT` were actually deleted before repopulating. # Checklist - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [x] This change is covered by an existing CI test or a new one has been added --- workflow/setup_expt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index e213394e20..494f5ded4d 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -7,14 +7,13 @@ import os import glob import shutil -import warnings from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS from hosts import Host from wxflow import parse_j2yaml from wxflow import AttrDict -from wxflow import to_datetime, to_timedelta, datetime_to_YMDH +from wxflow import to_datetime, datetime_to_YMDH _here = os.path.dirname(__file__) @@ -303,7 +302,7 @@ def query_and_clean(dirname, force_clean=False): if os.path.exists(dirname): print(f'\ndirectory already exists in {dirname}') if force_clean: - overwrite = True + overwrite = "YES" print(f'removing directory ........ {dirname}\n') else: overwrite = input('Do you wish to over-write [y/N]: ') From 0735fca219f7bc5a33379253166091af1db14f7d Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:41:32 -0400 Subject: [PATCH 08/21] Add a tool to run multiple YAML cases locally (#3004) This adds the script `generate_workflows.py`, which provides a flexible, multipurpose feature set to run suites of cases from YAMLs with minimal intervention. For most cases, this will allow users to easily run CI tests on a local platform, including updating the user's crontab automatically. Resolves #2989 --- .github/CODEOWNERS | 3 + workflow/generate_workflows.sh | 532 +++++++++++++++++++++++++++++++++ 2 files changed, 535 insertions(+) create mode 100755 workflow/generate_workflows.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e3521739c2..5068b961f7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -208,3 +208,6 @@ ush/python/pygfs/ufswm/gfs.py @aerorahul ush/python/pygfs/ufswm/ufs.py @aerorahul ush/python/pygfs/utils/__init__.py @aerorahul ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA + +# Specific workflow scripts +workflow/generate_workflows.sh @DavidHuber-NOAA diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh new file mode 100755 index 0000000000..ab40214655 --- /dev/null +++ b/workflow/generate_workflows.sh @@ -0,0 +1,532 @@ +#!/usr/bin/env bash + +### +function _usage() { + cat <<-EOF + This script automates the experiment setup process for the global workflow. + Options are also available to update submodules, build the workflow (with + specific build flags), specicy which YAMLs and YAML directory to run, and + whether to automatically update your crontab. + + Usage: generate_workflows.sh [OPTIONS] /path/to/RUNTESTS + or + RUNTESTS=/path/to/RUNTESTS generate_workflows.sh [OPTIONS] + + -H Root directory of the global workflow. + If not specified, then the directory is assumed to be one parent + directory up from this script's residing directory. + + -b Run build_all.sh with default flags + (build the UFS, UPP, UFS_Utils, and GFS-utils only + + -B "build flags" + Run build_all.sh with the build specified flags. Refer to + build_all.sh -h for a list of valid flags. + NOTE: the list of build flags MUST be in quotes. + + -u Update submodules before building and/or generating experiments. + + -y "list of YAMLs to run" + If this option is not specified, the default case (C48_ATM) will be + run. This option is overidden by -G or -E (see below). + Example: -y "C48_ATM C48_S2SW C96C48_hybatmDA" + + -Y /path/to/directory/with/YAMLs + If this option is not specified, then the \${HOMEgfs}/ci/cases/pr + directory is used. + + -G Run all valid GFS cases in the specified YAML directory. + If -b is specified, then "-g -u" (build the GSI and GDASApp) + will be passed to build_all.sh unless -B is also specified. + Note that these builds are disabled on some systems, which + will result in a warning from build_all.sh. + + -E Run all valid GEFS cases in the specified YAML directory. + If -b is specified, then "-w" will be passed to build_all.sh + unless -B is also specified. + + -S (Not yet supported!) + Run all valid SFS cases in the specified YAML directory. + + NOTES: + - Only one of -G -E or -S may be specified + - Valid cases are determined by the experiment:system key as + well as the skip_ci_on_hosts list in each YAML. + + -A "HPC account name" Set the HPC account name. + If this is not set, the default in + \$HOMEgfs/ci/platform/config.\$machine + will be used. + + -c Append the chosen set of tests to your existing crontab + If this option is not chosen, the new entries that would have been + written to your crontab will be printed to stdout. + NOTES: + - This option is not supported on Gaea. Instead, the output will + need to be written to scrontab manually. + - For Orion/Hercules, this option will not work unless run on + the [orion|hercules]-login-1 head node. + + -e "your@email.com" Email address to place in the crontab. + If this option is not specified, then the existing email address in + the crontab will be preserved. + + -v Verbose mode. Prints output of all commands to stdout. + + -V Very verbose mode. Passes -v to all commands and prints to stdout. + + -d Debug mode. Same as -V but also enables logging (set -x). + + -h Display this message. +EOF +} + +set -eu + +# Set default options +HOMEgfs="" +_specified_home=false +_build=false +_build_flags="" +_explicit_build_flags=false +_update_submods=false +declare -a _yaml_list=("C48_ATM") +_specified_yaml_list=false +_yaml_dir="" # Will be set based off of HOMEgfs if not specified explicitly +_specified_yaml_dir=false +_run_all_gfs=false +_run_all_gefs=false +_run_all_sfs=false +_hpc_account="" +_set_account=false +_update_cron=false +_email="" +_set_email=false +_verbose=false +_very_verbose=false +_verbose_flag="--" +_debug="false" +_cwd=$(pwd) +_runtests="${RUNTESTS:-${_runtests:-}}" +_nonflag_option_count=0 + +while [[ $# -gt 0 && "$1" != "--" ]]; do + while getopts ":H:bB:uy:Y:GESA:ce:vVdh" option; do + case "${option}" in + H) + HOMEgfs="${OPTARG}" + _specified_home=true + if [[ ! -d "${HOMEgfs}" ]]; then + echo "Specified HOMEgfs directory (${HOMEgfs}) does not exist" + exit 1 + fi + ;; + b) _build=true ;; + B) _build_flags="${OPTARG}" && _explicit_build_flags=true ;; + u) _update_submods=true ;; + y) # Start over with an empty _yaml_list + declare -a _yaml_list=() + for _yaml in ${OPTARG}; do + # Strip .yaml from the end of each and append to _yaml_list + _yaml_list+=("${_yaml//.yaml/}") + done + _specified_yaml_list=true + ;; + Y) _yaml_dir="${OPTARG}" && _specified_yaml_dir=true ;; + G) _run_all_gfs=true ;; + E) _run_all_gefs=true ;; + S) _run_all_sfs=true ;; + c) _update_cron=true ;; + e) _email="${OPTARG}" && _set_email=true ;; + v) _verbose=true ;; + V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;; + d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;; + h) _usage && exit 0 ;; + :) + echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" + _usage + ;; + *) + echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" + _usage + ;; + esac + done + + if [[ ${OPTIND:-0} -gt 0 ]]; then + shift $((OPTIND-1)) + fi + + while [[ $# -gt 0 && ! "$1" =~ ^- ]]; do + _runtests=${1} + (( _nonflag_option_count += 1 )) + if [[ ${_nonflag_option_count} -gt 1 ]]; then + echo "Too many arguments specified." + _usage + exit 2 + fi + shift + done +done + +function send_email() { + # Send an email to $_email. + # Only use this once we get to the long steps (building, etc) and on success. + _subject="${_subject:-generate_workflows.sh failure on ${machine}}" + _body="${1}" + + echo "${_body}" | mail -s "${_subject}" "${_email}" +} + +if [[ -z "${_runtests}" ]]; then + echo "Mising run directory (RUNTESTS) argument/environment variable." + sleep 2 + _usage + exit 3 +fi + +# Turn on logging if running in debug mode +if [[ "${_debug}" == "true" ]]; then + set -x +fi + +# Create the RUNTESTS directory +[[ "${_verbose}" == "true" ]] && printf "Creating RUNTESTS in %s\n\n" "${_runtests}" +if [[ ! -d "${_runtests}" ]]; then + set +e + if ! mkdir -p "${_runtests}" "${_verbose_flag}"; then + echo "Unable to create RUNTESTS directory: ${_runtests}" + echo "Rerun with -h for usage examples." + exit 4 + fi + set -e +else + echo "The RUNTESTS directory ${_runtests} already exists." + echo "Would you like to remove it?" + _attempts=0 + while read -r _from_stdin + do + if [[ "${_from_stdin^^}" =~ Y ]]; then + rm -rf "${_runtests}" + mkdir -p "${_runtests}" + break + elif [[ "${_from_stdin^^}" =~ N ]]; then + echo "Continuing without removing the directory" + break + else + (( _attempts+=1 )) + if [[ ${_attempts} == 3 ]]; then + echo "Exiting." + exit 99 + fi + echo "'${_from_stdin}' is not a valid choice. Please type Y or N" + fi + done +fi + +# Test if multiple "run_all" options were set +_count_run_alls=0 +[[ "${_run_all_gfs}" == "true" ]] && ((_count_run_alls+=1)) +[[ "${_run_all_gefs}" == "true" ]] && ((_count_run_alls+=1)) +[[ "${_run_all_sfs}" == "true" ]] && ((_count_run_alls+=1)) + +if (( _count_run_alls > 1 )) ; then + echo "Only one run all option (-G -E -S) may be specified" + echo "Rerun with just one option and/or with -h for usage examples" + exit 5 +fi + +# If -S is specified, exit (for now). +# TODO when SFS tests come online, enable this option. +if [[ "${_run_all_sfs}" == "true" ]]; then + echo "There are no known SFS tests at this time. Aborting." + echo "If you have prepared YAMLs for SFS cases, specify their" + echo "location and names without '-S', e.g." + echo "generate_workflows.sh -y \"C48_S2S_SFS\" -Y \"/path/to/yaml/directory\"" + exit 0 +fi + +# Set HOMEgfs if it wasn't set by the user +if [[ "${_specified_home}" == "false" ]]; then + script_relpath="$(dirname "${BASH_SOURCE[0]}")" + HOMEgfs="$(cd "${script_relpath}/.." && pwd)" + [[ "${_verbose}" == "true" ]] && printf "Setting HOMEgfs to %s\n\n" "${HOMEgfs}" +fi + +# Set the _yaml_dir to HOMEgfs/ci/cases/pr if not explicitly set +[[ "${_specified_yaml_dir}" == false ]] && _yaml_dir="${HOMEgfs}/ci/cases/pr" + +function select_all_yamls() +{ + # A helper function to select all of the YAMLs for a specified system (gfs, gefs, sfs) + + # This function is called if -G, -E, or -S are specified either with or without a + # specified YAML list. If a YAML list was specified, this function will remove any + # YAMLs in that list that are not for the specified system and issue warnings when + # doing so. + + _system="${1}" + _SYSTEM="${_system^^}" + + # Bash cannot return an array from a function and any edits are descoped at + # the end of the function, so use a nameref instead. + local -n _nameref_yaml_list='_yaml_list' + + if [[ "${_specified_yaml_list}" == false ]]; then + # Start over with an empty _yaml_list + _nameref_yaml_list=() + printf "Running all %s cases in %s\n\n" "${_SYSTEM}" "${_yaml_dir}" + _yaml_count=0 + + for _full_path in "${_yaml_dir}/"*.yaml; do + # Skip any YAML that isn't supported + if ! grep -l "system: *${_system}" "${_full_path}" >& /dev/null ; then continue; fi + + # Select only cases for the specified system + _yaml=$(basename "${_full_path}") + # Strip .yaml from the filename to get the case name + _yaml="${_yaml//.yaml/}" + _nameref_yaml_list+=("${_yaml}") + [[ "${_verbose}" == true ]] && echo "Found test ${_yaml//.yaml/}" + (( _yaml_count+=1 )) + done + + if [[ ${_yaml_count} -eq 0 ]]; then + read -r -d '' _message << EOM + "No YAMLs or ${_SYSTEM} were found in the directory (${_yaml_dir})!" + "Please check the directory/YAMLs and try again" +EOM + echo "${_message}" + if [[ "${_set_email}" == true ]]; then + send_email "${_message}" + fi + exit 6 + fi + else + # Check if the specified yamls are for the specified system + for i in "${!_nameref_yaml_list}"; do + _yaml="${_nameref_yaml_list[${i}]}" + _found=$(grep -l "system: *${system}" "${_yaml_dir}/${_yaml}.yaml") + if [[ -z "${_found}" ]]; then + echo "WARNING the yaml file ${_yaml_dir}/${_yaml}.yaml is not designed for the ${_SYSTEM} system" + echo "Removing this yaml from the set of cases to run" + unset '_nameref_yaml_list[${i}]' + # Sleep 2 seconds to give the user a moment to react + sleep 2s + fi + done + fi +} + +# Check if running all GEFS cases +if [[ "${_run_all_gefs}" == "true" ]]; then + # Append -w to build_all.sh flags if -E was specified + if [[ "${_explicit_build_flags}" == "false" && "${_build}" == "true" ]]; then + _build_flags="-w" + fi + + select_all_yamls "gefs" +fi + +# Check if running all SFS cases +if [[ "${_run_all_gfs}" == "true" ]]; then + # Append -g -u to build_all.sh flags if -G was specified + if [[ "${_explicit_build_flags}" == "false" && "${_build}" == "true" ]]; then + _build_flags="-g -u" + fi + + select_all_yamls "gfs" +fi + +# Loading modules sometimes raises unassigned errors, so disable checks +set +u +[[ "${_verbose}" == "true" ]] && printf "Loading modules\n\n" +[[ "${_debug}" == "true" ]] && set +x +if ! source "${HOMEgfs}/workflow/gw_setup.sh" >& stdout; then + cat stdout + echo "Failed to source ${HOMEgfs}/workflow/gw_setup.sh!" + exit 7 +fi +[[ "${_verbose}" == "true" ]] && cat stdout +rm -f stdout +[[ "${_debug}" == "true" ]] && set -x +set -u +machine=${MACHINE_ID} +. "${HOMEgfs}/ci/platforms/config.${machine}" + +# If _yaml_dir is not set, set it to $HOMEgfs/ci/cases/pr +if [[ -z ${_yaml_dir} ]]; then + _yaml_dir="${HOMEgfs}/ci/cases/pr" +fi + +# Update submodules if requested +if [[ "${_update_submods}" == "true" ]]; then + printf "Updating submodules\n\n" + _git_cmd="git submodule update --init --recursive -j 10" + if [[ "${_verbose}" == true ]]; then + ${_git_cmd} + else + if ! ${_git_cmd} 2> stderr 1> stdout; then + cat stdout stderr + read -r -d '' _message << EOM +The git command (${_git_cmd}) failed with a non-zero status +Messages from git: +EOM + _newline=$'\n' + _message="${_message}${_newline}$(cat stdout stderr)" + if [[ "${_set_email}" == true ]]; then + send_email "${_message}" + fi + echo "${_message}" + rm -f stdout stderr + exit 8 + fi + rm -f stdout stderr + fi +fi + +# Build the system if requested +if [[ "${_build}" == "true" ]]; then + printf "Building via build_all.sh %s\n\n" "${_build_flags}" + # Let the output of build_all.sh go to stdout regardless of verbose options + #shellcheck disable=SC2086,SC2248 + ${HOMEgfs}/sorc/build_all.sh ${_verbose_flag} ${_build_flags} +fi + +# Link the workflow silently unless there's an error +[[ "${_verbose}" == true ]] && printf "Linking the workflow\n\n" +if ! "${HOMEgfs}/sorc/link_workflow.sh" >& stdout; then + cat stdout + echo "link_workflow.sh failed!" + if [[ "${_set_email}" == true ]]; then + _stdout=$(cat stdout) + send_email "link_workflow.sh failed with the message"$'\n'"${_stdout}" + fi + rm -f stdout + exit 9 +fi +rm -f stdout + +# Configure the environment for running create_experiment.py +[[ "${_verbose}" == true ]] && printf "Setting up the environment to run create_experiment.py\n\n" +for i in "${!_yaml_list[@]}"; do + _yaml_file="${_yaml_dir}/${_yaml_list[${i}]}.yaml" + # Verify that the YAMLs are where we are pointed + if [[ ! -s "${_yaml_file}" ]]; then + echo "The YAML file ${_yaml_file} does not exist!" + echo "Please check the input yaml list and directory." + if [[ "${_set_email}" == true ]]; then + read -r -d '' _message << EOM + generate_workflows.sh failed to find one of the specified YAMLs (${_yaml_file}) + in the specified YAML directory (${_yaml_dir}). +EOM + send_email "${_message}" + fi + exit 10 + fi + + # Strip any unsupported tests + _unsupported_systems=$(sed '1,/skip_ci_on_hosts/ d' "${_yaml_file}") + + for _system in ${_unsupported_systems}; do + if [[ "${_system}" =~ ${machine} ]]; then + if [[ "${_specified_yaml_list}" == true ]]; then + printf "WARNING %s is unsupported on %s, removing from case list\n\n" "${_yaml}" "${machine}" + if [[ "${_set_email}" == true ]]; then + _final_message="${_final_message:-}"$'\n'"The specified YAML case ${_yaml} is not supported on ${machine} and was skipped." + fi + # Sleep so the user has a moment to notice + sleep 2s + fi + unset '_yaml_list[${i}]' + break + fi + done +done + +# Update the account if specified +[[ "${_set_account}" == true ]] && export HPC_ACCOUNT=${_hpc_account} && \ + [[ "${_verbose}" == true ]] && printf "Setting HPC account to %s\n\n" "${HPC_ACCOUNT}" + +# Create the experiments +rm -f "tests.cron" "${_verbose_flag}" +echo "Running create_experiment.py for ${#_yaml_list[@]} cases" + +[[ "${_verbose}" == true ]] && printf "Selected cases: %s\n\n" "${_yaml_list[*]}" +for _case in "${_yaml_list[@]}"; do + [[ "${_verbose}" == false ]] && echo "${_case}" + _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite" + if [[ "${_verbose}" == true ]]; then + pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} + else + if ! pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then + _output=$(cat stdout stderr) + _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status. Output:" + _message="${_message}"$'\n'"${_output}" + if [[ "${_set_email}" == true ]]; then + send_email "${_message}" + fi + echo "${_message}" + rm -f stdout stderr + exit 11 + fi + rm -f stdout stderr + fi + grep "${_case}" "${_runtests}/EXPDIR/${_case}/${_case}.crontab" >> tests.cron +done +echo + +# Update the cron +if [[ "${_update_cron}" == "true" ]]; then + printf "Updating the existing crontab\n\n" + echo + rm -f existing.cron final.cron "${_verbose_flag}" + touch existing.cron final.cron + + # disable -e in case crontab is empty + set +e + crontab -l > existing.cron + set -e + + if [[ "${_debug}" == "true" ]]; then + echo "Existing crontab: " + echo "#######################" + cat existing.cron + echo "#######################" + fi + + if [[ "${_set_email}" == "true" ]]; then + # Replace the existing email in the crontab + [[ "${_verbose}" == "true" ]] && printf "Updating crontab email to %s\n\n" "${_email}" + sed -i "/^MAILTO/d" existing.cron + echo "MAILTO=\"${_email}\"" >> final.cron + fi + + cat existing.cron tests.cron >> final.cron + + if [[ "${_verbose}" == "true" ]]; then + echo "Setting crontab to:" + echo "#######################" + cat final.cron + echo "#######################" + fi + + crontab final.cron +else + _message="Add the following to your crontab or scrontab to start running:" + _cron_tests=$(cat tests.cron) + _message="${_message}"$'\n'"${_cron_tests}" + echo "${_message}" + if [[ "${_set_email}" == true ]]; then + final_message="${final_message:-}"$'\n'"${_message}" + fi +fi + +# Cleanup +[[ "${_debug}" == "false" ]] && rm -f final.cron existing.cron tests.cron "${_verbose_flag}" + +echo "Success!!" +if [[ "${_set_email}" == true ]]; then + final_message=$'Success!\n'"${final_message:-}" + _subject="generate_workflow.sh completed successfully" send_email "${final_message}" +fi From 76befe142ec6fa8cd51f68497e57c5dfce11c384 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:44:41 -0400 Subject: [PATCH 09/21] Hotfix: Fix generate_workflows.sh optional build flags (#3024) # Description The build flags used when invoking build_all.sh from generate_workflows.sh did were after the verbose flag, which caused them to be ignored if any of the verbose flags (-v, -V, -d) were not set when calling generate_workflows.sh as the `build_all.sh` verbose flag would be set to `--`. This fixes the issue by changing the order of the flags sent to `build_all.sh`. # Type of change - [x] Bug fix (fixes something broken) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? Build test on Hercules. # Checklist - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes --- workflow/generate_workflows.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index ab40214655..c4f89eef6e 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -2,7 +2,7 @@ ### function _usage() { - cat <<-EOF + cat << EOF This script automates the experiment setup process for the global workflow. Options are also available to update submodules, build the workflow (with specific build flags), specicy which YAMLs and YAML directory to run, and @@ -204,8 +204,7 @@ else echo "The RUNTESTS directory ${_runtests} already exists." echo "Would you like to remove it?" _attempts=0 - while read -r _from_stdin - do + while read -r _from_stdin; do if [[ "${_from_stdin^^}" =~ Y ]]; then rm -rf "${_runtests}" mkdir -p "${_runtests}" @@ -390,7 +389,7 @@ if [[ "${_build}" == "true" ]]; then printf "Building via build_all.sh %s\n\n" "${_build_flags}" # Let the output of build_all.sh go to stdout regardless of verbose options #shellcheck disable=SC2086,SC2248 - ${HOMEgfs}/sorc/build_all.sh ${_verbose_flag} ${_build_flags} + ${HOMEgfs}/sorc/build_all.sh ${_build_flags} ${_verbose_flag} fi # Link the workflow silently unless there's an error From 720eb4c06c7ec325b2175ab5f85177225b81da87 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Tue, 22 Oct 2024 12:33:53 -0400 Subject: [PATCH 10/21] Replace gfs_cyc with an interval (#2928) # Description To facilitate longer and more flexible GFS cadences, the `gfs_cyc` variable is replaced with a specified interval. Up front, this is reflected in a change in the arguments for setup_exp to: ``` --interval ``` Where `n_hours` is the interval (in hours) between gfs forecasts. `n_hours` must be a multiple of 6. If 0, no gfs will be run (only gdas; only valid for cycled mode). The default value is 6 (every cycle). (This is a change from current behavior of 24.) In cycled mode, there is an additional argument to control which cycle will be the first gfs cycle: ``` --sdate_gfs ``` The default if not provided is `--idate` + 6h (first full cycle). This is the same as current behavior when `gfs_cyc` is 6, but may vary from current behavior for other cadences. As part of this change, some of the validation of the dates has been added. `--edate` has also been made optional and defaults to `--idate` if not provided. During `config.base` template-filling, `INTERVAL_GFS` (renamed from `STEP_GFS`) is defined as `--interval` and `SDATE_GFS as `--sdate_gfs`. Some changes were necessary to the gfs verification (metp) job, as `gfs_cyc` was being used downstream by verif-global. That has been removed, and instead workflow will be responsible for only running metp on the correct cycles. This also removes "do nothing" metp tasks that exit immediately, because only the last GFS cycle in a day would actually process verification. Now, metp has its own cycledef and will (a) always runs at 18z, regardless of whether gfs is running at 18z or not, if the interval is less than 24h; (b) use the same cycledef as gfs if the interval is 24h or greater. This is simpler than trying to determine the last gfs cycle of a day when it could change from day to day. To facilitate this change, support for the undocumented rocoto dependency tag `taskvalid` is added, as the metp task needs to know whether the cycle has a gfsarch task or not. metp will trigger on gfsarch completing (as before), or look backwards for the last gfsarch to exist. Additionally, a couple EE2 issues with the metp job are resolved (even though it is not run in ops): - verif-global update replaced `$CDUMP` with `$RUN` - `$DATAROOT` is no longer redefined in the metp job Also corrects some dependency issues with the extractvars job for replay and the replay CI test. Depends on NOAA-EMC/EMC_verif-global#137 Resolves #260 Refs #1299 --------- Co-authored-by: David Huber --- ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C48_S2SWA_gefs.yaml | 2 +- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmaerosnowDA.yaml | 2 +- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml | 5 +- ci/cases/pr/C96_atm3DVar.yaml | 2 +- ci/cases/pr/C96_atm3DVar_extended.yaml | 2 +- ci/cases/weekly/C384C192_hybatmda.yaml | 2 +- ci/cases/weekly/C384_atm3DVar.yaml | 2 +- docs/source/jobs.rst | 2 +- docs/source/setup.rst | 18 ++--- gempak/ush/gfs_meta_comp.sh | 2 +- gempak/ush/gfs_meta_mar_comp.sh | 2 +- jobs/JGFS_ATMOS_VERIFICATION | 6 -- parm/config/gefs/config.base | 8 +-- parm/config/gfs/config.base | 9 +-- parm/config/gfs/config.resources | 4 +- parm/config/gfs/config.wave | 2 +- scripts/exgfs_aero_init_aerosol.py | 16 ++--- sorc/verif-global.fd | 2 +- workflow/applications/applications.py | 17 +---- workflow/applications/gefs.py | 1 - workflow/applications/gfs_cycled.py | 50 +------------ workflow/applications/gfs_forecast_only.py | 2 +- workflow/rocoto/gefs_tasks.py | 2 +- workflow/rocoto/gefs_xml.py | 12 ++-- workflow/rocoto/gfs_cycled_xml.py | 33 +++++++-- workflow/rocoto/gfs_forecast_only_xml.py | 37 +++++++--- workflow/rocoto/gfs_tasks.py | 83 +++++++++++++--------- workflow/rocoto/rocoto.py | 22 ++++++ workflow/rocoto/tasks.py | 3 +- workflow/setup_expt.py | 52 +++++++++++--- 34 files changed, 227 insertions(+), 183 deletions(-) diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml index d97c9567e9..99ba7c3661 100644 --- a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 384 resdetocean: 0.25 nens: 0 - gfs_cyc: 4 + interval: 6 start: cold comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml index 98f0fcfadb..f39031f1a1 100644 --- a/ci/cases/pr/C48_S2SWA_gefs.yaml +++ b/ci/cases/pr/C48_S2SWA_gefs.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 5.0 resensatmos: 48 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index e1b76f0db8..2de5fea7ff 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2021032412 edate: 2021032418 nens: 0 - gfs_cyc: 0 + interval: 0 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml index 7617e39217..b527903d69 100644 --- a/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_hybatmDA.yaml @@ -14,6 +14,6 @@ arguments: idate: 2021122018 edate: 2021122106 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index 7387e55b24..be5ad32238 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2021122012 edate: 2021122100 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index b1566d77a0..41a8baa725 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2024022318 edate: 2024022406 nens: 2 - gfs_cyc: 1 + interval: 24 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml diff --git a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml index 1475e81ea0..7118dde53f 100644 --- a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml +++ b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 1.0 resensatmos: 96 nens: 2 - gfs_cyc: 1 + interval: 6 start: warm comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR @@ -17,3 +17,6 @@ arguments: edate: 2020110100 yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_replay_ci.yaml icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610 + +skip_ci_on_hosts: + - wcoss2 diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml index e9e6c2b31c..fc09beeacf 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/ci/cases/pr/C96_atm3DVar.yaml @@ -12,7 +12,7 @@ arguments: idate: 2021122018 edate: 2021122106 nens: 0 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml index cdf69f04e0..8ab67a750e 100644 --- a/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -12,7 +12,7 @@ arguments: idate: 2021122018 edate: 2021122118 nens: 0 - gfs_cyc: 4 + interval: 6 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/ci/cases/weekly/C384C192_hybatmda.yaml index 131ada95d5..6053f73124 100644 --- a/ci/cases/weekly/C384C192_hybatmda.yaml +++ b/ci/cases/weekly/C384C192_hybatmda.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/weekly/C384_atm3DVar.yaml b/ci/cases/weekly/C384_atm3DVar.yaml index 40487f3b47..1a14059ab1 100644 --- a/ci/cases/weekly/C384_atm3DVar.yaml +++ b/ci/cases/weekly/C384_atm3DVar.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 0 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index 0e3700bf20..2cdecb01de 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -8,7 +8,7 @@ GFS Configuration The sequence of jobs that are run for an end-to-end (analysis+forecast+post processing+verification) GFS configuration using the Global Workflow is shown above. The system utilizes a collection of scripts that perform the tasks for each step. -For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. As with the operational system, the gdas runs for each cycle (00, 06, 12, and 18 UTC), however, to save time and space in experiments, the gfs (right side of the diagram) is initially setup to run for only the 00 UTC cycle (See the "run GFS this cycle?" portion of the diagram). The option to run the GFS for all four cycles is available (see the ``gfs_cyc`` variable in configuration file). +For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. An experimental run is different from operations in the following ways: diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 1715899927..e7d5323e40 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -32,7 +32,7 @@ The following command examples include variables for reference but users should :: cd workflow - ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] + ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--interval $INTERVAL_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] where: @@ -51,12 +51,12 @@ where: * ``$START`` is the start type (warm or cold [default]) * ``$IDATE`` is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) - * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete [default: $IDATE] * ``$PSLOT`` is the name of your experiment [default: test] * ``$CONFIGDIR`` is the path to the ``/config`` folder under the copy of the system you're using [default: $TOP_OF_CLONE/parm/config/] * ``$RESDETATMOS`` is the resolution of the atmosphere component of the system (i.e. 768 for C768) [default: 384] * ``$RESDETOCEAN`` is the resolution of the ocean component of the system (i.e. 0.25 for 1/4 degree) [default: 0.; determined based on atmosphere resolution] - * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * ``$INTERVAL_GFS`` is the forecast interval in hours [default: 6] * ``$COMROOT`` is the path to your experiment output directory. Your ``ROTDIR`` (rotating com directory) will be created using ``COMROOT`` and ``PSLOT``. [default: $HOME (but do not use default due to limited space in home directories normally, provide a path to a larger scratch space)] * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] @@ -67,7 +67,7 @@ Atm-only: :: cd workflow - ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --gfs_cyc 4 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir + ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --interval 6 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir Coupled: @@ -144,7 +144,8 @@ The following command examples include variables for reference but users should :: cd workflow - ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] + ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] + [--interval $INTERVAL_GFS] [--sdate_gfs $SDATE_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--resensatmos $RESENSATMOS] [--nens $NENS] [--run $RUN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] [--icsdir $ICSDIR] @@ -163,9 +164,10 @@ where: - S2SWA: atm-ocean-ice-wave-aerosols * ``$IDATE`` is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) - * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete [default: $IDATE] * ``$START`` is the start type (warm or cold [default]) - * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * ``$INTERVAL_GFS`` is the forecast interval in hours [default: 6] + * ``$SDATE_GFS`` cycle to begin GFS forecast [default: $IDATE + 6] * ``$RESDETATMOS`` is the resolution of the atmosphere component of the deterministic forecast [default: 384] * ``$RESDETOCEAN`` is the resolution of the ocean component of the deterministic forecast [default: 0.; determined based on atmosphere resolution] * ``$RESENSATMOS`` is the resolution of the atmosphere component of the ensemble forecast [default: 192] @@ -184,7 +186,7 @@ Example: :: cd workflow - ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --gfs_cyc 4 + ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --interval 6 Example ``setup_expt.py`` on Orion: diff --git a/gempak/ush/gfs_meta_comp.sh b/gempak/ush/gfs_meta_comp.sh index 36d18d8659..38c15a60c7 100755 --- a/gempak/ush/gfs_meta_comp.sh +++ b/gempak/ush/gfs_meta_comp.sh @@ -24,7 +24,7 @@ device="nc | ${metaname}" export COMIN="gfs.multi" mkdir "${COMIN}" -for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do +for cycle in $(seq -f "%02g" -s ' ' 0 "${INTERVAL_GFS}" "${cyc}"); do YMD=${PDY} HH=${cycle} GRID="1p00" declare_from_tmpl gempak_dir:COM_ATMOS_GEMPAK_TMPL for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do file_out="${COMIN}/$(basename "${file_in}")" diff --git a/gempak/ush/gfs_meta_mar_comp.sh b/gempak/ush/gfs_meta_mar_comp.sh index d25fc0dc9a..91f8a48876 100755 --- a/gempak/ush/gfs_meta_mar_comp.sh +++ b/gempak/ush/gfs_meta_mar_comp.sh @@ -15,7 +15,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl export COMIN="gfs.multi" mkdir -p "${COMIN}" -for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do +for cycle in $(seq -f "%02g" -s ' ' 0 "${INTERVAL_GFS}" "${cyc}"); do YMD=${PDY} HH=${cycle} GRID="1p00" declare_from_tmpl gempak_dir:COM_ATMOS_GEMPAK_TMPL for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do file_out="${COMIN}/$(basename "${file_in}")" diff --git a/jobs/JGFS_ATMOS_VERIFICATION b/jobs/JGFS_ATMOS_VERIFICATION index 48133364e5..fde0d73b1e 100755 --- a/jobs/JGFS_ATMOS_VERIFICATION +++ b/jobs/JGFS_ATMOS_VERIFICATION @@ -16,12 +16,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "metp" -c "base metp" ## METPCASE : METplus verification use case (g2g1 | g2o1 | pcp1) ############################################################### -# TODO: This should not be permitted as DATAROOT is set at the job-card level. -# TODO: DATAROOT is being used as DATA in metp jobs. This should be rectified in metp. -# TODO: The temporary directory is DATA and is created at the top of the J-Job. -# TODO: remove this line -export DATAROOT=${DATA} - VDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${VRFYBACK_HRS} hours") export VDATE=${VDATE:0:8} diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 6cf8488f91..05aabaa323 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -227,7 +227,8 @@ export FHOUT_OCN=3 export FHOUT_ICE=3 # GFS cycle info -export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ # set variables needed for use with REPLAY ICs export REPLAY_ICS=@REPLAY_ICS@ @@ -255,11 +256,6 @@ export FHOUT_WAV=3 export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 export FHMAX_WAV=${FHMAX_GFS} -if (( gfs_cyc != 0 )); then - export STEP_GFS=$(( 24 / gfs_cyc )) -else - export STEP_GFS="0" -fi export ILPOST=1 # gempak output frequency up to F120 export FHMIN_ENKF=${FHMIN_GFS} diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 7fa8245057..ccb05abe88 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -283,7 +283,8 @@ export FHOUT_ICE=3 export EUPD_CYC="@EUPD_CYC@" # GFS cycle info -export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ # GFS output and frequency export FHMIN_GFS=0 @@ -302,11 +303,7 @@ export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 export FHMAX_WAV=${FHMAX:-9} export FHMAX_WAV_GFS=${FHMAX_GFS} -if (( gfs_cyc != 0 )); then - export STEP_GFS=$(( 24 / gfs_cyc )) -else - export STEP_GFS="0" -fi + # TODO: Change gempak to use standard out variables (#2348) export ILPOST=${FHOUT_HF_GFS} # gempak output frequency up to F120 if (( FHMAX_HF_GFS < 120 )); then diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 26f6126773..79dbb487db 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -988,8 +988,8 @@ case ${step} in threads_per_task=1 walltime_gdas="03:00:00" walltime_gfs="06:00:00" - ntasks=4 - tasks_per_node=4 + ntasks=1 + tasks_per_node=1 export memory="80G" ;; diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index db4eb9f708..ea68508547 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -117,7 +117,7 @@ if [[ "${RUN}" == "gdas" ]]; then export WAVNCYC=4 export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=48 # RTOFS forecasts only out to 8 days -elif [[ ${gfs_cyc} -ne 0 ]]; then +elif (( INTERVAL_GFS > 0 )); then export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=192 # RTOFS forecasts only out to 8 days else diff --git a/scripts/exgfs_aero_init_aerosol.py b/scripts/exgfs_aero_init_aerosol.py index aed6b88647..bc4e495e42 100755 --- a/scripts/exgfs_aero_init_aerosol.py +++ b/scripts/exgfs_aero_init_aerosol.py @@ -11,13 +11,13 @@ --------- This script requires the following environment variables be set beforehand: -CDATE: Initial time in YYYYMMDDHH format -STEP_GFS: Forecast cadence (frequency) in hours -FHMAX_GFS: Forecast length in hours -RUN: Forecast phase (gfs or gdas). Currently always expected to be gfs. -ROTDIR: Rotating (COM) directory -USHgfs: Path to global-workflow `ush` directory -PARMgfs: Path to global-workflow `parm` directory +CDATE: Initial time in YYYYMMDDHH format +INTERVAL_GFS: Forecast cadence (frequency) in hours +FHMAX_GFS: Forecast length in hours +RUN: Forecast phase (gfs or gdas). Currently always expected to be gfs. +ROTDIR: Rotating (COM) directory +USHgfs: Path to global-workflow `ush` directory +PARMgfs: Path to global-workflow `parm` directory Additionally, the following data files are used: @@ -66,7 +66,7 @@ def main() -> None: # Read in environment variables and make sure they exist cdate = get_env_var("CDATE") - incr = int(get_env_var('STEP_GFS')) + incr = int(get_env_var('INTERVAL_GFS')) fcst_length = int(get_env_var('FHMAX_GFS')) run = get_env_var("RUN") rot_dir = get_env_var("ROTDIR") diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index e7e6bc4358..b2ee80cac7 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit e7e6bc43584e0b8911819b8f875cc8ee747db76d +Subproject commit b2ee80cac7921a3016fa5a857cc58acfccc4baea diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index a694129e38..ecd320d708 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -68,6 +68,7 @@ def __init__(self, conf: Configuration) -> None: self.nens = base.get('NMEM_ENS', 0) self.fcst_segments = base.get('FCST_SEGMENTS', None) + self.interval_gfs = to_timedelta(f"{base.get('INTERVAL_GFS')}H") if not AppConfig.is_monotonic(self.fcst_segments): raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') @@ -109,9 +110,6 @@ def _init_finalize(self, conf: Configuration): # Save base in the internal state since it is often needed base = self.configs['_no_run']['base'] - # Get more configuration options into the class attributes - self.gfs_cyc = base.get('gfs_cyc') - # Get task names for the application self.task_names = self.get_task_names() @@ -199,19 +197,6 @@ def get_task_names(self, run="_no_run") -> Dict[str, List[str]]: ''' pass - @staticmethod - def get_gfs_interval(gfs_cyc: int) -> timedelta: - """ - return interval in hours based on gfs_cyc - """ - - gfs_internal_map = {'1': '24H', '2': '12H', '4': '6H'} - - try: - return to_timedelta(gfs_internal_map[str(gfs_cyc)]) - except KeyError: - raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') - @staticmethod def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: """ diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index c15786a2e8..9d1d5c3dc4 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -42,7 +42,6 @@ def _get_app_configs(self): def _update_base(base_in): base_out = base_in.copy() - base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) base_out['RUN'] = 'gefs' return base_out diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 19f4dd607b..da78166ede 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -128,7 +128,7 @@ def _get_app_configs(self): @staticmethod def _update_base(base_in): - return GFSCycledAppConfig.get_gfs_cyc_dates(base_in) + return base_in def get_task_names(self): """ @@ -297,7 +297,7 @@ def get_task_names(self): tasks['enkfgdas'] = enkfgdas_tasks # Add RUN=gfs tasks if running early cycle - if self.gfs_cyc > 0: + if self.interval_gfs > to_timedelta("0H"): tasks['gfs'] = gfs_tasks if self.do_hybvar and 'gfs' in self.eupd_runs: @@ -307,49 +307,3 @@ def get_task_names(self): tasks['enkfgfs'] = enkfgfs_tasks return tasks - - @staticmethod - def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: - """ - Generate GFS dates from experiment dates and gfs_cyc choice - """ - - base_out = base.copy() - - sdate = base['SDATE'] - edate = base['EDATE'] - base_out['INTERVAL'] = to_timedelta(f"{base['assim_freq']}H") - - # Set GFS cycling dates - gfs_cyc = base['gfs_cyc'] - if gfs_cyc != 0: - interval_gfs = AppConfig.get_gfs_interval(gfs_cyc) - hrinc = 0 - hrdet = 0 - if gfs_cyc == 1: - hrinc = 24 - sdate.hour - hrdet = edate.hour - elif gfs_cyc == 2: - if sdate.hour in [0, 12]: - hrinc = 12 - elif sdate.hour in [6, 18]: - hrinc = 6 - if edate.hour in [6, 18]: - hrdet = 6 - elif gfs_cyc == 4: - hrinc = 6 - sdate_gfs = sdate + timedelta(hours=hrinc) - edate_gfs = edate - timedelta(hours=hrdet) - if sdate_gfs > edate: - print('W A R N I N G!') - print('Starting date for GFS cycles is after Ending date of experiment') - print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') - print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') - gfs_cyc = 0 - - base_out['gfs_cyc'] = gfs_cyc - base_out['SDATE_GFS'] = sdate_gfs - base_out['EDATE_GFS'] = edate_gfs - base_out['INTERVAL_GFS'] = interval_gfs - - return base_out diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 93551ac0cc..fb1d2cdb8f 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -78,7 +78,7 @@ def _get_app_configs(self): def _update_base(base_in): base_out = base_in.copy() - base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) + base_out['RUN'] = 'gfs' return base_out diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 955f631c8e..e9338c90df 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -472,7 +472,7 @@ def wavepostpnt(self): def extractvars(self): deps = [] if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': 'wave_post_grid_mem#member#'} + dep_dict = {'type': 'task', 'name': 'gefs_wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ocean: dep_dict = {'type': 'metatask', 'name': 'gefs_ocean_prod_#member#'} diff --git a/workflow/rocoto/gefs_xml.py b/workflow/rocoto/gefs_xml.py index b25a73fa6c..a5dfd5140e 100644 --- a/workflow/rocoto/gefs_xml.py +++ b/workflow/rocoto/gefs_xml.py @@ -14,19 +14,19 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE'] + sdate = self._base['SDATE_GFS'] edate = self._base['EDATE'] - interval = self._base.get('INTERVAL_GFS', to_timedelta('24H')) + interval = self._app_config.interval_gfs sdate_str = sdate.strftime("%Y%m%d%H%M") edate_str = edate.strftime("%Y%m%d%H%M") interval_str = timedelta_to_HMS(interval) strings = [] strings.append(f'\t{sdate_str} {edate_str} {interval_str}') - sdate = sdate + interval - if sdate <= edate: - sdate_str = sdate.strftime("%Y%m%d%H%M") - strings.append(f'\t{sdate_str} {edate_str} {interval_str}') + date2 = sdate + interval + if date2 <= edate: + date2_str = date2.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_str} {edate_str} {interval_str}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_cycled_xml.py b/workflow/rocoto/gfs_cycled_xml.py index afd663c337..eef77ba7fc 100644 --- a/workflow/rocoto/gfs_cycled_xml.py +++ b/workflow/rocoto/gfs_cycled_xml.py @@ -24,19 +24,38 @@ def get_cycledefs(self): sdate_str = sdate.strftime("%Y%m%d%H%M") strings.append(f'\t{sdate_str} {edate_str} {interval_str}') - if self._app_config.gfs_cyc != 0: + interval_gfs = self._app_config.interval_gfs + + if interval_gfs > to_timedelta("0H"): sdate_gfs = self._base['SDATE_GFS'] - edate_gfs = self._base['EDATE_GFS'] - interval_gfs = self._base['INTERVAL_GFS'] + edate_gfs = sdate_gfs + ((edate - sdate_gfs) // interval_gfs) * interval_gfs sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") interval_gfs_str = timedelta_to_HMS(interval_gfs) strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') - sdate_gfs = sdate_gfs + interval_gfs - sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") - if sdate_gfs <= edate_gfs: - strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') + date2_gfs = sdate_gfs + interval_gfs + date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") + if date2_gfs <= edate_gfs: + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + if self._base['DO_METP']: + if interval_gfs < to_timedelta('24H'): + # Run verification at 18z, no matter what if there is more than one gfs per day + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + else: + # Use same cycledef as gfs if there is no more than one per day + sdate_metp_str = sdate_gfs_str + edate_metp_str = edate_gfs_str + interval_metp_str = interval_gfs_str + + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') + strings.append(f'\t{edate_gfs_str} {edate_gfs_str} 24:00:00') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/workflow/rocoto/gfs_forecast_only_xml.py index cf53e685e9..a4d5b0878b 100644 --- a/workflow/rocoto/gfs_forecast_only_xml.py +++ b/workflow/rocoto/gfs_forecast_only_xml.py @@ -12,15 +12,36 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE'] - edate = self._base['EDATE'] - interval = self._base.get('INTERVAL_GFS', to_timedelta('24H')) + sdate_gfs = self._base['SDATE_GFS'] + edate_gfs = self._base['EDATE'] + interval_gfs = self._app_config.interval_gfs strings = [] - strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') - - sdate = sdate + interval - if sdate <= edate: - strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') + sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") + edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") + interval_gfs_str = timedelta_to_HMS(interval_gfs) + strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + date2 = sdate_gfs + interval_gfs + if date2 <= edate_gfs: + date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + if self._base['DO_METP']: + if interval_gfs < to_timedelta('24H'): + # Run verification at 18z, no matter what if there is more than one gfs per day + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + else: + # Use same cycledef as gfs if there is no more than one per day + sdate_metp_str = sdate_gfs_str + edate_metp_str = edate_gfs_str + interval_metp_str = interval_gfs_str + + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 7c56f25583..82dfb9f1d4 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1,6 +1,6 @@ from applications.applications import AppConfig from rocoto.tasks import Tasks -from wxflow import timedelta_to_HMS +from wxflow import timedelta_to_HMS, to_timedelta import rocoto.rocoto as rocoto import numpy as np @@ -39,7 +39,6 @@ def stage_ic(self): def prep(self): dump_suffix = self._base["DUMP_SUFFIX"] - gfs_cyc = self._base["gfs_cyc"] dmpdir = self._base["DMPDIR"] atm_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'RUN': 'gdas'}) dump_path = self._template_to_rocoto_cycstring(self._base["COM_OBSDMP_TMPL"], @@ -48,10 +47,10 @@ def prep(self): gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False deps = [] - dep_dict = {'type': 'metatask', 'name': 'gdas_atmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdas_atmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) data = f'{atm_hist_path}/gdas.t@Hz.atmf009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) data = f'{dump_path}/{self.run}.t@Hz.updated.status.tm00.bufr_d' dep_dict = {'type': 'data', 'data': data} @@ -59,7 +58,7 @@ def prep(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.run in ['gfs'] and gfs_enkf and self.app_config.interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('prep') @@ -89,7 +88,7 @@ def waveinit(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -189,7 +188,7 @@ def anal(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: @@ -255,7 +254,7 @@ def analcalc(self): dep_dict = {'type': 'task', 'name': f'{self.run}_sfcanl'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar and self.run in ['gdas']: - dep_dict = {'type': 'task', 'name': 'enkfgdas_echgres', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': 'enkfgdas_echgres', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -330,17 +329,17 @@ def atmanlinit(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) - gfs_cyc = self._base["gfs_cyc"] + interval_gfs = self._base["INTERVAL_GFS"] gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.run in ['gfs'] and gfs_enkf and interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('atmanlinit') @@ -482,7 +481,7 @@ def aeroanlgenb(self): def aeroanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': 'gdas_aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': 'gdas_aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) @@ -514,7 +513,7 @@ def aeroanlvar(self): deps = [] dep_dict = { 'type': 'task', 'name': f'gdas_aeroanlgenb', - 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", + 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}", } deps.append(rocoto.add_dependency(dep_dict)) dep_dict = { @@ -618,7 +617,7 @@ def esnowrecen(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_snowanl'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{self.run}_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -644,7 +643,7 @@ def prepoceanobs(self): deps = [] data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -671,7 +670,7 @@ def marinebmat(self): deps = [] data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -699,7 +698,7 @@ def marineanlinit(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmat'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'gdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -878,9 +877,9 @@ def _fcst_forecast_only(self): # Calculate offset based on RUN = gfs | gdas interval = None if self.run in ['gfs']: - interval = self._base['INTERVAL_GFS'] + interval = to_timedelta(f"{self._base['INTERVAL_GFS']}H") elif self.run in ['gdas']: - interval = self._base['INTERVAL'] + interval = self._base['assim_freq'] offset = timedelta_to_HMS(-interval) deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_aerosol_init'} @@ -1835,14 +1834,27 @@ def metp(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_arch'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + if self.app_config.interval_gfs < to_timedelta('24H'): + n_lookback = self.app_config.interval_gfs // to_timedelta('6H') + for lookback in range(1, n_lookback + 1): + deps2 = [] + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_arch', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) + for lookback2 in range(1, lookback): + offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback2}H')) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': offset} + deps2.append(rocoto.add_dependency(dep_dict)) + + offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback}H')) + dep_dict = {'type': 'task', 'name': f'{self.run}_arch', 'offset': offset} + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='and', dep=deps2)) + + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) metpenvars = self.envars.copy() - if self.app_config.mode in ['cycled']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H"), - 'EDATE_GFS': self._base.get('EDATE_GFS').strftime("%Y%m%d%H")} - elif self.app_config.mode in ['forecast-only']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE').strftime("%Y%m%d%H")} + metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H"), + 'EDATE_GFS': self._base.get('EDATE').strftime("%Y%m%d%H")} metpenvar_dict['METPCASE'] = '#metpcase#' for key, value in metpenvar_dict.items(): metpenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -1858,7 +1870,7 @@ def metp(self): 'resources': resources, 'dependency': dependencies, 'envars': metpenvars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': 'metp,last_gfs', 'command': f'{self.HOMEgfs}/jobs/rocoto/metp.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2331,8 +2343,13 @@ def cleanup(self): deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_metp and self.run in ['gfs']: + deps2 = [] + # taskvalid only handles regular tasks, so just check the first metp job exists + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_metpg2g1', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': f'{self.run}_metp'} - deps.append(rocoto.add_dependency(dep_dict)) + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='or', dep=deps2)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2358,7 +2375,7 @@ def eobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_prep'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2468,7 +2485,7 @@ def atmensanlinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2495,7 +2512,7 @@ def atmensanlobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2521,7 +2538,7 @@ def atmensanlsol(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2547,7 +2564,7 @@ def atmensanlletkf(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2576,7 +2593,7 @@ def atmensanlfv3inc(self): else: dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlletkf'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index 2a20820da8..7bacf99829 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -183,6 +183,7 @@ def add_dependency(dep_dict: Dict[str, Any]) -> str: 'metatask': _add_task_tag, 'data': _add_data_tag, 'cycleexist': _add_cycle_tag, + 'taskvalid': _add_taskvalid_tag, 'streq': _add_streq_tag, 'strneq': _add_streq_tag, 'sh': _add_sh_tag} @@ -296,6 +297,27 @@ def _add_cycle_tag(dep_dict: Dict[str, Any]) -> str: return string +def _add_taskvalid_tag(dep_dict: Dict[str, Any]) -> str: + """ + create a validtask tag + :param dep_dict: dependency key-value parameters + :type dep_dict: dict + :return: Rocoto validtask dependency + :rtype: str + """ + + dep_type = dep_dict.get('type', None) + dep_name = dep_dict.get('name', None) + + if dep_name is None: + msg = f'a {dep_type} name is necessary for {dep_type} dependency' + raise KeyError(msg) + + string = f'<{dep_type} task="{dep_name}"/>' + + return string + + def _add_streq_tag(dep_dict: Dict[str, Any]) -> str: """ create a simple string comparison tag diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 8a32827377..92ceea73aa 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -57,7 +57,8 @@ def __init__(self, app_config: AppConfig, run: str) -> None: self.nmem = int(self._base['NMEM_ENS_GFS']) else: self.nmem = int(self._base['NMEM_ENS']) - self._base['cycle_interval'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gdas'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gfs'] = to_timedelta(f'{self._base["INTERVAL_GFS"]}H') self.n_tiles = 6 # TODO - this needs to be elsewhere diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 494f5ded4d..f32203e600 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -7,13 +7,14 @@ import os import glob import shutil -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS +import warnings +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS, ArgumentTypeError from hosts import Host from wxflow import parse_j2yaml from wxflow import AttrDict -from wxflow import to_datetime, datetime_to_YMDH +from wxflow import to_datetime, to_timedelta, datetime_to_YMDH _here = os.path.dirname(__file__) @@ -115,7 +116,8 @@ def edit_baseconfig(host, inputs, yaml_dict): "@COMROOT@": inputs.comroot, "@EXP_WARM_START@": is_warm_start, "@MODE@": inputs.mode, - "@gfs_cyc@": inputs.gfs_cyc, + "@INTERVAL_GFS@": inputs.interval, + "@SDATE_GFS@": datetime_to_YMDH(inputs.sdate_gfs), "@APP@": inputs.app, "@NMEM_ENS@": getattr(inputs, 'nens', 0) } @@ -185,6 +187,19 @@ def input_args(*argv): ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW', 'S2SWA'] + def _validate_interval(interval_str): + err_msg = f'must be a non-negative integer multiple of 6 ({interval_str} given)' + try: + interval = int(interval_str) + except ValueError: + raise ArgumentTypeError(err_msg) + + # This assumes the gdas frequency (assim_freq) is 6h + # If this changes, the modulus needs to as well + if interval < 0 or interval % 6 != 0: + raise ArgumentTypeError(err_msg) + return interval + def _common_args(parser): parser.add_argument('--pslot', help='parallel experiment name', type=str, required=False, default='test') @@ -198,7 +213,8 @@ def _common_args(parser): type=str, required=False, default=os.getenv('HOME')) parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: to_datetime(dd)) - parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd)) + parser.add_argument('--edate', help='end date experiment', required=False, type=lambda dd: to_datetime(dd)) + parser.add_argument('--interval', help='frequency of forecast (in hours); must be a multiple of 6', type=_validate_interval, required=False, default=6) parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='') parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', action='store_true', required=False) @@ -218,8 +234,7 @@ def _gfs_args(parser): def _gfs_cycled_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='cycles to run forecast', type=int, - choices=[0, 1, 2, 4], default=1, required=False) + parser.add_argument('--sdate_gfs', help='date to start GFS', type=lambda dd: to_datetime(dd), required=False, default=None) return parser def _gfs_or_gefs_ensemble_args(parser): @@ -232,8 +247,6 @@ def _gfs_or_gefs_ensemble_args(parser): def _gfs_or_gefs_forecast_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='Number of forecasts per day', type=int, - choices=[1, 2, 4], default=1, required=False) return parser def _gefs_args(parser): @@ -290,7 +303,28 @@ def _gefs_args(parser): for subp in [gefsforecasts]: subp = _gefs_args(subp) - return parser.parse_args(list(*argv) if len(argv) else None) + inputs = parser.parse_args(list(*argv) if len(argv) else None) + + # Validate dates + if inputs.edate is None: + inputs.edate = inputs.idate + + if inputs.edate < inputs.idate: + raise ArgumentTypeError(f'edate ({inputs.edate}) cannot be before idate ({inputs.idate})') + + # For forecast-only, GFS starts in the first cycle + if not hasattr(inputs, 'sdate_gfs'): + inputs.sdate_gfs = inputs.idate + + # For cycled, GFS starts after the half-cycle + if inputs.sdate_gfs is None: + inputs.sdate_gfs = inputs.idate + to_timedelta("6H") + + if inputs.interval > 0: + if inputs.sdate_gfs < inputs.idate or inputs.sdate_gfs > inputs.edate: + raise ArgumentTypeError(f'sdate_gfs ({inputs.sdate_gfs}) must be between idate ({inputs.idate}) and edate ({inputs.edate})') + + return inputs def query_and_clean(dirname, force_clean=False): From 5cc20ec811f0f2a77b8a0e2616b86332a3eb48cf Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:48:57 -0400 Subject: [PATCH 11/21] Document the generate_workflows.sh script (#3028) This adds readthedocs documentation for the new generate_workflows.sh script to the testing section. It also fixes a few small existing issues in the documentation. --- docs/source/clone.rst | 2 +- docs/source/development.rst | 25 ++++++++++++++++++++++++- docs/source/hpc.rst | 4 ++-- docs/source/init.rst | 2 +- docs/source/setup.rst | 4 ++-- docs/source/wave.rst | 6 +++--- 6 files changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/source/clone.rst b/docs/source/clone.rst index c365f0ed0a..d3f81f2e47 100644 --- a/docs/source/clone.rst +++ b/docs/source/clone.rst @@ -9,7 +9,7 @@ Quick Instructions Quick clone/build/link instructions (more detailed instructions below). .. note:: - Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development.rst`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from. + Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from. Clone the `global-workflow` and `cd` into the `sorc` directory: diff --git a/docs/source/development.rst b/docs/source/development.rst index 4739d2b602..fc07f3f55e 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -12,6 +12,7 @@ Code managers * Kate Friedman - @KateFriedman-NOAA / kate.friedman@noaa.gov * Walter Kolczynski - @WalterKolczynski-NOAA / walter.kolczynski@noaa.gov + * David Huber - @DavidHuber-NOAA / david.huber@noaa.gov .. _development: @@ -70,7 +71,29 @@ The following steps should be followed in order to make changes to the develop b Development Tools ================= -See the ``/test`` folder in global-workflow for available development and testing tools. +Two sets of testing are available for use by developers. The first is the capability to run continuous integration tests locally and the second are a set of comparison tools. + +--------------------------- +Continuous Integration (CI) +--------------------------- + +The global workflow comes fitted with a suite of system tests that run various types of workflow. These tests are commonly run for pull requests before they may be merged into the develop branch. At a minimum, developers are expected to run the CI test(s) that will be impacted by their changes on at least one platform. + +The commonly run tests are written in YAML format and can be found in the ``ci/cases/pr`` directory. The ``workflow/generate_workflows.sh`` tool is available to aid running these cases. See the help documentation by running ``./generate_workflows.sh -h``. The script has the capability to prepare the EXPDIR and COMROOT directories for a specified or implied suite of CI tests (see :doc:`setup` for details on these directories). The script also has options to automatically build and run all tests for a given system (i.e. GFS or GEFS and a placeholder for SFS). For instance, to build the workflow and run all of the GFS tests, one would execute + +:: + + cd workflow + ./generate_workflows.sh -A "your_hpc_account" -b -G -c /path/to/root/directory + +where: + + * ``-A`` is used to specify the HPC (slurm or PBS) account to use + * ``-b`` indicates that the workflow should be built fresh + * ``-G`` specifies that all of the GFS cases should be run (this also influences the build flags to use) + * ``-c`` tells the tool to append the rocotorun commands for each experiment to your crontab + +More details on how to use the tool are provided by running ``generate_workflows.sh -h``. ---------------- Comparison Tools diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index 643cffdef0..e83851b1a2 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -90,9 +90,9 @@ Optimizing the global workflow on S4 The S4 cluster is relatively small and so optimizations are recommended to improve cycled runtimes. Please contact Innocent Souopgui (innocent.souopgui@noaa.gov) if you are planning on running a cycled experiment on this system to obtain optimized configuration files. -======================================== +================================================== Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4) -======================================== +================================================== Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components: diff --git a/docs/source/init.rst b/docs/source/init.rst index aa71e4e294..bd1bc1d2ce 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -301,7 +301,7 @@ Operations/production output location on HPSS: /NCEPPROD/hpssprod/runhistory/rh | | | | | | | gfs.t. ``hh`` z.sfcanl.nc | | | +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ -| v16.2[3]+ ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos| +| v16.2[3]+ ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs\_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos| | | | | | | | gfs.t. ``hh`` z.sfcanl.nc | | | +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ diff --git a/docs/source/setup.rst b/docs/source/setup.rst index e7d5323e40..0916033cbd 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -1,3 +1,5 @@ +.. _experiment-setup: + ================ Experiment Setup ================ @@ -179,8 +181,6 @@ where: * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] * ``$ICSDIR`` is the path to the ICs for your run if generated separately. [default: None] -.. [#] More Coupled configurations in cycled mode are currently under development and not yet available - Example: :: diff --git a/docs/source/wave.rst b/docs/source/wave.rst index 7b4f7471b8..56aa34ce3b 100644 --- a/docs/source/wave.rst +++ b/docs/source/wave.rst @@ -79,7 +79,7 @@ While the creation of these files are generally considered out of scope of this * Instructions for creating mesh.${WAVEGRID}.nc can be found at https://ufs-weather-model.readthedocs.io/en/latest/InputsOutputs.html#ww3 * The ww3_gint.WHTGRIDINT.bin.${WAVEGRID} can be created by running the ww3_gint routine as desired and then saved. -Once the new fix files have been created, :ref:`open an issue to have the master fix file directory updated`. This is a separate step than the process to update the workflow below. +Once the new fix files have been created, `open an issue to have the master fix file directory updated `. This is a separate step than the process to update the workflow below. ******************************** Updating Config and Script Files @@ -87,8 +87,8 @@ Updating Config and Script Files You will need to update the following files: -* parm/config/*/config.ufs -* parm/config/*/config.wave +* parm/config/\*/config.ufs +* parm/config/\*/config.wave * scripts/exgfs_wave_post_gridded_sbs.sh You will need to add the following files: From 3b2c4e27cedb71cdb9fa3718ac2f1ca85f7b467c Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 24 Oct 2024 11:45:36 -0400 Subject: [PATCH 12/21] Quick updated to Jenkins (health check) launch script (#3033) # Description This is a quick hotfix for small discrepancy in the Jenkins health check launch script that runs in cron: `gh -> "${GH}"` # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How is this tested Ran in cron on Hercules and Orion Co-authored-by: tmcguinness --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 183e671b9d..eb78d3b1ef 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -106,7 +106,7 @@ export GH="${HOME}/bin/gh" [[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin" ${GH} --version -check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true +check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true if [[ "${check_mark}" != "✓" ]]; then echo "gh not authenticating with emcbot token" exit 1 From 58737cb2f097d7d0ef0535b38fa0cff0f714a654 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 24 Oct 2024 15:35:08 -0400 Subject: [PATCH 13/21] CI update to shell environment with HOMEgfs to HOME_GFS for systems that need the path (#3013) # Description This PR is setting a shell environment variable **HOME_GFS** to point to the path **HOMEgfs** for systems that need to setup the runtime environment in their respective `.bashrc` files. For example: ``` # Added for Global Workflow if [ -n "${HOME_GFS}"]; then export MODULE_GWSETUP_PATH="${HOME_GFS}/modulefiles" module use "${MODULE_GWSETUP_PATH}" module load "module_gwsetup.gaea" fi ``` # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [x] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? This PR itself will be ran on Gaea directly when `$HOMEgfs/workflow/gw_setup.sh` script for loading the runtime modules is operable on that system. # Checklist - [ ] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added - [ ] I have made corresponding changes to the system documentation if necessary --- ci/Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 2654adba29..1a63e104bb 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -104,6 +104,7 @@ pipeline { catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc sh(script: "mkdir -p ${HOMEgfs}") ws(HOMEgfs) { if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs @@ -194,10 +195,11 @@ pipeline { stage("Create ${caseName}") { catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { + def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" system = yaml_case.experiment.system - def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" try { error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() @@ -213,6 +215,7 @@ pipeline { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { script { HOMEgfs = "${CUSTOM_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") @@ -273,6 +276,7 @@ pipeline { agent { label NodeName[machine].toLowerCase() } steps { script { + env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc sh(script: """ labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name') for label in \$labels; do From 39d0b8b12ae3afd781e7c0bcb83e4573f1a4dfc7 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Fri, 25 Oct 2024 09:22:42 -0400 Subject: [PATCH 14/21] Add ability to add tag to pslots with generate_workflows (#3036) # Description `generate_workflows.sh` would only use the case name as the pslot, leading to conflicts if you try to run two different sets at the same time. Now there is a new option that allows a 'tag' to the case name when determining the pslots: ``` generate_workflows.sh -t tag: string to be added to end of case name ``` For example, this: ``` generate_workflows.sh -t test ``` Will result in pslots of `C48_ATM_test`, `C48_S2SW_test`, etc. This is similar to how the CI system appends the hash to the pslots, but allows flexibility since this is a command-line tool. # Type of change - [ ] Bug fix (fixes something broken) - [x] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? - Ran `generate_workflows.sh` both with and without the new `-t` option. # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added - [x] Any new scripts have been added to the .github/CODEOWNERS file with owners - [x] I have made corresponding changes to the system documentation if necessary --- workflow/generate_workflows.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index c4f89eef6e..c2565140d3 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -71,6 +71,9 @@ function _usage() { If this option is not specified, then the existing email address in the crontab will be preserved. + -t Add a 'tag' to the end of the case names in the pslots to distinguish + pslots between multiple sets of tests. + -v Verbose mode. Prints output of all commands to stdout. -V Very verbose mode. Passes -v to all commands and prints to stdout. @@ -101,6 +104,7 @@ _hpc_account="" _set_account=false _update_cron=false _email="" +_tag="" _set_email=false _verbose=false _very_verbose=false @@ -111,7 +115,7 @@ _runtests="${RUNTESTS:-${_runtests:-}}" _nonflag_option_count=0 while [[ $# -gt 0 && "$1" != "--" ]]; do - while getopts ":H:bB:uy:Y:GESA:ce:vVdh" option; do + while getopts ":H:bB:uy:Y:GESA:ce:t:vVdh" option; do case "${option}" in H) HOMEgfs="${OPTARG}" @@ -138,6 +142,7 @@ while [[ $# -gt 0 && "$1" != "--" ]]; do S) _run_all_sfs=true ;; c) _update_cron=true ;; e) _email="${OPTARG}" && _set_email=true ;; + t) _tag="_${OPTARG}" ;; v) _verbose=true ;; V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;; d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;; @@ -454,11 +459,12 @@ echo "Running create_experiment.py for ${#_yaml_list[@]} cases" [[ "${_verbose}" == true ]] && printf "Selected cases: %s\n\n" "${_yaml_list[*]}" for _case in "${_yaml_list[@]}"; do [[ "${_verbose}" == false ]] && echo "${_case}" + _pslot="${_case}${_tag}" _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite" if [[ "${_verbose}" == true ]]; then - pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} + pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} else - if ! pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then + if ! pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then _output=$(cat stdout stderr) _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status. Output:" _message="${_message}"$'\n'"${_output}" @@ -471,7 +477,7 @@ for _case in "${_yaml_list[@]}"; do fi rm -f stdout stderr fi - grep "${_case}" "${_runtests}/EXPDIR/${_case}/${_case}.crontab" >> tests.cron + grep "${_pslot}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab" >> tests.cron done echo From 2fdfaa5a39a71544a907bdcc2dbd9559acf60a2b Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Sat, 26 Oct 2024 18:00:16 -0400 Subject: [PATCH 15/21] Update gdas.cd (#2978) Updates to the `gdas.cd` #. @RussTreadon-NOAA will submit a PR in the GDASApp, we'll update the gdas.cd # in this branch after the GDASApp PR is merged. In the mean time, could somebody review the few simple code changes that are needed to run with the new #? - [x] Depends on GDASApp PR [#1310](https://github.com/NOAA-EMC/GDASApp/pull/1310) - [x] Depends on g-w issue [#3012](https://github.com/NOAA-EMC/global-workflow/issues/3012) --------- Co-authored-by: Rahul Mahajan Co-authored-by: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Co-authored-by: RussTreadon-NOAA --- ci/Jenkinsfile | 2 +- ci/cases/pr/C96C48_hybatmaerosnowDA.yaml | 1 + sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_analysis.py | 1 + ush/python/pygfs/utils/marine_da_utils.py | 33 +++++++++++++++++++++++ versions/fix.ver | 2 +- 6 files changed, 38 insertions(+), 3 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 1a63e104bb..639e0ff223 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -5,7 +5,7 @@ def cases = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index be5ad32238..57d0989ae3 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -18,6 +18,7 @@ arguments: yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml skip_ci_on_hosts: + - wcoss2 - orion - gaea - hercules diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 55e895f1dc..764f58cebd 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 +Subproject commit 764f58cebdf64f3695d89538994a50183e5884d9 diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 4e4311b906..e7b7b5e948 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -210,6 +210,7 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['cyc'] = os.getenv('cyc') envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER envconfig_jcb['obs_list'] = ['adt_rads_all'] + envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES)) # Write obs_list_short save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index e1b2ac2d4d..50d9d84e86 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -166,3 +166,36 @@ def clean_empty_obsspaces(config, target, app='var'): # save cleaned yaml save_as_yaml(config, target) + + +@logit(logger) +def get_mom6_levels(ocnres: str) -> int: + """ + Temporary function that returns the number of vertical levels in MOM6 given the horizontal resolution. + This is requiered by the diffusion saber block that now makes use of oops::util::FieldSetHelpers::writeFieldSet + and requires the number of levels in the configuration. I have been told this will be changed in the future. + + Parameters + ----------- + ocnres: str + Input resolution for ocean in str format. e.g. '500', '100', '050', '025' + + Returns + ------- + nlev: int + number of levels in the ocean model given an input resolution + """ + + # Currently implemented resolutions + ocnres_to_nlev = { + '500': 25, + '100': 75, + '050': 75, + '025': 75 + } + try: + nlev = ocnres_to_nlev.get(ocnres) + except KeyError: + raise KeyError("FATAL ERROR: Invalid ocnres value. Aborting.") + + return nlev diff --git a/versions/fix.ver b/versions/fix.ver index 7c18bea081..b175791196 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -8,7 +8,7 @@ export cice_ver=20240416 export cpl_ver=20230526 export datm_ver=20220805 export gdas_crtm_ver=20220805 -export gdas_fv3jedi_ver=20220805 +export gdas_fv3jedi_ver=20241022 export gdas_soca_ver=20240802 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 From b56f633d4a0feaf65955e19f01d1e888b4120db4 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sun, 27 Oct 2024 21:20:54 -0400 Subject: [PATCH 16/21] CI jekninsfile update hotfix (#3038) This PR is a needed hotfix to the Jenkinsfile pipeline script that fixes a bug with the matrix control variable name clash with the variable **system** for the two build types. --- ci/Jenkinsfile | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 639e0ff223..6a2e064be0 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -101,7 +101,7 @@ pipeline { stages { stage('Building') { steps { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc @@ -173,9 +173,10 @@ pipeline { } if (system == 'gfs') { cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() + echo "Cases to run: ${cases}" } } - } + } } } } @@ -193,22 +194,20 @@ pipeline { def parallelStages = cases.collectEntries { caseName -> ["${caseName}": { stage("Create ${caseName}") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis - env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc - sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") - def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" - system = yaml_case.experiment.system - env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" - try { - error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() - } catch (Exception error_create) { - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) - error("Case ${caseName} failed to create experment directory") - } + script { + sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + def build_system = yaml_case.experiment.system + def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}" // local HOMEgfs is used to populate the XML on per system basis + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc + env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" + try { + error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() + } catch (Exception error_create) { + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) + error("Case ${caseName} failed to create experment directory") } - } + } } stage("Running ${caseName}") { @@ -219,8 +218,10 @@ pipeline { def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + def build_system = yaml_case.experiment.system try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}") + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") } catch (Exception error_experment) { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") ws(CUSTOM_WORKSPACE) { @@ -271,7 +272,6 @@ pipeline { } } - stage( '5. FINALIZE' ) { agent { label NodeName[machine].toLowerCase() } steps { From 00f567cd721f343e9c13a416ac455e1f8f612586 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 29 Oct 2024 01:56:37 -0400 Subject: [PATCH 17/21] GitHub Actions Pipeline Updates for Self-Hosted Runners on PW (#3018) Updates to the GItHub Self-hosted Runner Pipelines for Parallel Works: - User now inputs PR number when dispatching the CI workflow - User can also select which operating system to test on during the transition to Rocky 8 **NOTE:** When this pipeline is dispatched it should be run with [PR 3023](https://github.com/NOAA-EMC/global-workflow/pull/3023) to demonstrate that it can pass an end-to-end case. --- .../{pw_aws_centos.yaml => pw_aws_ci.yaml} | 89 ++++++++++++++----- 1 file changed, 68 insertions(+), 21 deletions(-) rename .github/workflows/{pw_aws_centos.yaml => pw_aws_ci.yaml} (51%) diff --git a/.github/workflows/pw_aws_centos.yaml b/.github/workflows/pw_aws_ci.yaml similarity index 51% rename from .github/workflows/pw_aws_centos.yaml rename to .github/workflows/pw_aws_ci.yaml index 76258c3044..245e219dd4 100644 --- a/.github/workflows/pw_aws_centos.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -1,7 +1,4 @@ -name: gw-ci-aws-centos - -on: [workflow_dispatch] - +name: gw-ci-aws # TEST_DIR contains 2 directories; # 1. HOMEgfs: clone of the global-workflow # 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments @@ -10,33 +7,73 @@ on: [workflow_dispatch] # ├── HOMEgfs # └── RUNTESTS # ├── COMROT -# │   └── ${pslot} +# │ └── ${pslot} # └── EXPDIR # └── ${pslot} + +on: + workflow_dispatch: + inputs: + pr_number: + description: 'Pull Request Number (use 0 for non-PR)' + required: true + default: '0' + os: + description: 'Operating System' + required: true + type: choice + options: + - rocky + - centos + env: TEST_DIR: ${{ github.workspace }}/${{ github.run_id }} MACHINE_ID: noaacloud jobs: + fetch-branch: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUBTOKEN }} + outputs: + branch: ${{ steps.get-branch.outputs.branch }} + steps: + - name: Fetch branch name for PR + id: get-branch + run: | + pr_number=${{ github.event.inputs.pr_number }} + repo=${{ github.repository }} + if [ "$pr_number" -eq "0" ]; then + branch=${{ github.event.inputs.ref }} + else + branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') + fi + echo "::set-output name=branch::$branch" + checkout: - runs-on: [self-hosted, aws, parallelworks, centos] + needs: fetch-branch + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} timeout-minutes: 600 - steps: - - name: Checkout global-workflow uses: actions/checkout@v4 with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ needs.fetch-branch.outputs.branch }} build-link: - runs-on: [self-hosted, aws, parallelworks, centos] needs: checkout - + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} steps: - - name: Build components run: | cd ${{ env.TEST_DIR }}/HOMEgfs/sorc @@ -48,12 +85,15 @@ jobs: ./link_workflow.sh create-experiments: - needs: checkout - runs-on: [self-hosted, aws, parallelworks, centos] + needs: build-link + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} strategy: matrix: case: ["C48_ATM"] - steps: - name: Create Experiments ${{ matrix.case }} env: @@ -68,9 +108,12 @@ jobs: run-experiments: needs: create-experiments - runs-on: [self-hosted, aws, parallelworks, centos] + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} strategy: - max-parallel: 2 matrix: case: ["C48_ATM"] steps: @@ -81,9 +124,13 @@ jobs: clean-up: needs: run-experiments - runs-on: [self-hosted, aws, parallelworks, centos] + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} steps: - - name: Clean-up + - name: Clean up workspace run: | - cd ${{ github.workspace }} - rm -rf ${{ github.run_id }} + echo "Cleaning up workspace" + rm -rf ${{ env.TEST_DIR }} From fc2c5ea7e3ddde5312370922b3ab51a8d33c22c5 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Wed, 30 Oct 2024 07:15:37 -0400 Subject: [PATCH 18/21] Disabling hyper-threading (#2965) Hera, Hercules, and Orion (and possibly Jet) enable hyper-threading by default. This should be disabled explicitly by adding the `sbatch/srun` flag `--hint=nomultithread`. Resolves #2863 --- env/HERA.env | 4 ++-- env/HERCULES.env | 4 ++-- env/ORION.env | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/env/HERA.env b/env/HERA.env index 09743967b5..80cd7cddaf 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -9,7 +9,7 @@ fi step=$1 -export launcher="srun -l --export=ALL" +export launcher="srun -l --export=ALL --hint=nomultithread" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" #export POSTAMBLE_CMD='report-mem' @@ -50,7 +50,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then export POE="NO" export BACK="NO" export sys_tp="HERA" - export launcher_PREP="srun" + export launcher_PREP="srun --hint=nomultithread" elif [[ "${step}" = "prepsnowobs" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index 9ec112c699..bed1d11281 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -9,7 +9,7 @@ fi step=$1 -export launcher="srun -l --export=ALL" +export launcher="srun -l --export=ALL --hint=nomultithread" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment @@ -48,7 +48,7 @@ case ${step} in export POE="NO" export BACK=${BACK:-"YES"} export sys_tp="HERCULES" - export launcher_PREP="srun" + export launcher_PREP="srun --hint=nomultithread" ;; "prepsnowobs") diff --git a/env/ORION.env b/env/ORION.env index 3b8053d060..45fd607aa5 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -9,7 +9,7 @@ fi step=$1 -export launcher="srun -l --export=ALL" +export launcher="srun -l --export=ALL --hint=nomultithread" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment @@ -45,7 +45,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then export POE="NO" export BACK=${BACK:-"YES"} export sys_tp="ORION" - export launcher_PREP="srun" + export launcher_PREP="srun --hint=nomultithread" elif [[ "${step}" = "prepsnowobs" ]]; then From 5bb3f867e1f7a530986b9f291b7ec9d4d0cb1387 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Wed, 30 Oct 2024 10:58:18 -0400 Subject: [PATCH 19/21] Fix wave restarts and GEFS FHOUT/FHMAX (#3009) # Description Fixes some issues that were preventing wave restarts from operating correctly. First, the wave restart files were not being correctly linked from `$DATA` to `$DATArestart`. The files are placed in the root of `$DATA` instead of in `${DATA}/WAVE_RESTART`, so now links for the individual files are created. Second, the incorrect filenames were being searches for and copied as part of a rerun. Filenames were geared towards multigrid waves, which use the grid names, but single grid just uses a `ww3`. Since multigrid waves are deprecated in workflow and will soon be removed (#2637), these were updated only supporting the single-grid option. These fixes allow forecast segments (and emergency restarts) to work correctly when waves are on. Additionally, the `FHMAX` (and `FHOUT`) for perturbed GEFS members was being overwritten by `config.efcs` due to remnant code from when it was copied from the EnKF version. This interfered with the segment setting for those GEFS members. Resolves #3001 # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? - S2SW forecast-only test with segments on Hercules # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [x] This change is covered by an existing CI test or a new one has been added - [x] I have made corresponding changes to the system documentation if necessary --------- Co-authored-by: Kate.Friedman Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- parm/config/gefs/config.efcs | 14 +----------- parm/config/gefs/config.fcst | 3 ++- parm/config/gfs/config.fcst | 3 ++- parm/stage/wave.yaml.j2 | 6 +++++- ush/forecast_det.sh | 9 +++----- ush/forecast_postdet.sh | 42 ++++++++++++++++++++---------------- ush/forecast_predet.sh | 6 +++--- 7 files changed, 40 insertions(+), 43 deletions(-) diff --git a/parm/config/gefs/config.efcs b/parm/config/gefs/config.efcs index 9bd55afa54..0086121450 100644 --- a/parm/config/gefs/config.efcs +++ b/parm/config/gefs/config.efcs @@ -26,18 +26,6 @@ source "${EXPDIR}/config.ufs" ${string} # Get task specific resources source "${EXPDIR}/config.resources" efcs -# nggps_diag_nml -export FHOUT=${FHOUT_ENKF:-3} -if [[ "${RUN}" == "enkfgfs" ]]; then - export FHOUT=${FHOUT_ENKF_GFS:-${FHOUT}} -fi - -# model_configure -export FHMAX=${FHMAX_ENKF:-9} -if [[ "${RUN}" == "enkfgfs" ]]; then - export FHMAX=${FHMAX_ENKF_GFS:-${FHMAX}} -fi - # Stochastic physics parameters (only for ensemble forecasts) export DO_SKEB="YES" export SKEB="0.8,-999,-999,-999,-999" @@ -74,6 +62,6 @@ if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then else export ODA_INCUPD="False" fi -export restart_interval="${restart_interval_enkfgfs:-12}" +export restart_interval="${restart_interval_gfs:-12}" echo "END: config.efcs" diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index efdedb24f4..b2a9c10afe 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -35,7 +35,8 @@ IFS=', ' read -ra segments <<< "${FCST_SEGMENTS}" # Determine MIN and MAX based on the forecast segment export FHMIN=${segments[${FCST_SEGMENT}]} export FHMAX=${segments[${FCST_SEGMENT}+1]} -# Cap other FHMAX variables at FHMAX for the segment +# Cap other FH variables at FHMAX for the segment +export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV )) export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS )) export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV )) # shellcheck disable=SC2153 diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index da336ff73b..571e6cafb5 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -38,7 +38,8 @@ case ${RUN} in # Determine MIN and MAX based on the forecast segment export FHMIN=${segments[${FCST_SEGMENT}]} export FHMAX=${segments[${FCST_SEGMENT}+1]} - # Cap other FHMAX variables at FHMAX for the segment + # Cap other FH variables at FHMAX for the segment + export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV )) export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS )) export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV )) # shellcheck disable=SC2153 diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 index d610430bc7..2788a24343 100644 --- a/parm/stage/wave.yaml.j2 +++ b/parm/stage/wave.yaml.j2 @@ -9,5 +9,9 @@ wave: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"] + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".restart." ~ waveGRD) %} + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}/{{ m_prefix }}.restart.ww3"] + {% else %} + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.ww3", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"] + {% endif %} {% endfor %} # mem loop diff --git a/ush/forecast_det.sh b/ush/forecast_det.sh index 603447f612..72064ac7f5 100755 --- a/ush/forecast_det.sh +++ b/ush/forecast_det.sh @@ -93,12 +93,9 @@ UFS_det(){ # Check for WW3 restart availability if [[ "${cplwav}" == ".true." ]]; then - local ww3_grid - for ww3_grid in ${waveGRD} ; do - if [[ ! -f "${DATArestart}/WW3_RESTART/${rdate:0:8}.${rdate:8:2}0000.restart.${ww3_grid}" ]]; then - ww3_rst_ok="NO" - fi - done + if [[ ! -f "${DATArestart}/WW3_RESTART/${rdate:0:8}.${rdate:8:2}0000.restart.ww3" ]]; then + ww3_rst_ok="NO" + fi fi # Collective check diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 58755d41d9..288b251aa8 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -326,7 +326,7 @@ FV3_out() { WW3_postdet() { echo "SUB ${FUNCNAME[0]}: Linking input data for WW3" - local ww3_grid + local ww3_grid first_ww3_restart_out ww3_restart_file # Copy initial condition files: if [[ "${warm_start}" == ".true." ]]; then local restart_date restart_dir @@ -338,29 +338,35 @@ WW3_postdet() { restart_dir="${COMIN_WAVE_RESTART_PREV}" fi echo "Copying WW3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'" - local ww3_restart_file - for ww3_grid in ${waveGRD} ; do - ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.${ww3_grid}" - if [[ ! -f "${ww3_restart_file}" ]]; then - echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!" - if [[ "${RERUN}" == "YES" ]]; then - # In the case of a RERUN, the WW3 restart file is required - echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!" - exit 1 - fi - fi - if [[ "${waveMULTIGRID}" == ".true." ]]; then - ${NCP} "${ww3_restart_file}" "${DATA}/restart.${ww3_grid}" \ - || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 ) + ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.ww3" + if [[ -f "${ww3_restart_file}" ]]; then + ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \ + || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 ) + else + if [[ "${RERUN}" == "YES" ]]; then + # In the case of a RERUN, the WW3 restart file is required + echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!" + exit 1 else - ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \ - || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 ) + echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!" fi - done + fi + + first_ww3_restart_out=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H) else # cold start echo "WW3 will start from rest!" + first_ww3_restart_out="${model_start_date_current_cycle}" fi # [[ "${warm_start}" == ".true." ]] + # Link restart files + local ww3_restart_file + # Use restart_date if it was determined above, otherwise use initialization date + for (( vdate = first_ww3_restart_out; vdate <= forecast_end_cycle; + vdate = $(date --utc -d "${vdate:0:8} ${vdate:8:2} + ${restart_interval} hours" +%Y%m%d%H) )); do + ww3_restart_file="${vdate:0:8}.${vdate:8:2}0000.restart.ww3" + ${NLN} "${DATArestart}/WW3_RESTART/${ww3_restart_file}" "${ww3_restart_file}" + done + # Link output files local wavprfx="${RUN}wave${WAV_MEMBER:-}" if [[ "${waveMULTIGRID}" == ".true." ]]; then diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 5aa9dc9ac7..d359a86622 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -556,10 +556,10 @@ WW3_predet(){ echo "SUB ${FUNCNAME[0]}: WW3 before run type determination" if [[ ! -d "${COMOUT_WAVE_HISTORY}" ]]; then mkdir -p "${COMOUT_WAVE_HISTORY}"; fi - if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}" ; fi + if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}"; fi - if [[ ! -d "${DATArestart}/WAVE_RESTART" ]]; then mkdir -p "${DATArestart}/WAVE_RESTART"; fi - ${NLN} "${DATArestart}/WAVE_RESTART" "${DATA}/restart_wave" + if [[ ! -d "${DATArestart}/WW3_RESTART" ]]; then mkdir -p "${DATArestart}/WW3_RESTART"; fi + # Wave restarts are linked in postdet to only create links for files that will be created # Files from wave prep and wave init jobs # Copy mod_def files for wave grids From 0b3304eac1cbbe27e54978681da143820fbb11c0 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:25:12 -0400 Subject: [PATCH 20/21] Add run and finalize methods to marine LETKF task (#2944) Adds run and finalize methods to marine LETKF task, experiment yaml for gw-ci in GDASApp, workflow additions, removes bugs found on the way, and completes the bulk of the work on LETKF task. Conversion of fields to increments pending. Partially resolves NOAA-EMC/GDASApp#1091 and NOAA-EMC/GDASApp#1251 Mutual dependency with GDASApp PR NOAA-EMC/GDASApp#1287 and IC fix file issue https://github.com/NOAA-EMC/global-workflow/pull/2944#issue-2537157488 --------- Co-authored-by: Walter Kolczynski - NOAA Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/cases/pr/C48mx500_hybAOWCDA.yaml | 26 +++++++ env/HERA.env | 6 +- env/ORION.env | 6 +- env/WCOSS2.env | 6 +- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 25 ++++-- .../{marineanalletkf.sh => marineanlletkf.sh} | 2 +- parm/config/gfs/config.com | 2 + parm/config/gfs/config.marineanalletkf | 18 ----- parm/config/gfs/config.marineanlletkf | 20 +++++ parm/config/gfs/config.resources | 2 +- parm/stage/ocean_ens_perturbations.yaml.j2 | 2 +- ...f.py => exglobal_marine_analysis_letkf.py} | 0 ush/forecast_postdet.sh | 13 ++-- ush/python/pygfs/task/marine_bmat.py | 2 +- ush/python/pygfs/task/marine_letkf.py | 78 +++++++++++++++---- workflow/applications/gfs_cycled.py | 4 +- workflow/rocoto/gfs_tasks.py | 26 +++++++ workflow/rocoto/tasks.py | 2 +- 18 files changed, 177 insertions(+), 63 deletions(-) create mode 100644 ci/cases/pr/C48mx500_hybAOWCDA.yaml rename jobs/rocoto/{marineanalletkf.sh => marineanlletkf.sh} (95%) delete mode 100644 parm/config/gfs/config.marineanalletkf create mode 100644 parm/config/gfs/config.marineanlletkf rename scripts/{exgdas_global_marine_analysis_letkf.py => exglobal_marine_analysis_letkf.py} (100%) diff --git a/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/ci/cases/pr/C48mx500_hybAOWCDA.yaml new file mode 100644 index 0000000000..036aa8ca60 --- /dev/null +++ b/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -0,0 +1,26 @@ +experiment: + system: gfs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: S2S + resdetatmos: 48 + resdetocean: 5.0 + resensatmos: 48 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610 + idate: 2021032412 + edate: 2021032418 + nens: 3 + interval: 0 + start: warm + yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml + +skip_ci_on_hosts: + - wcoss2 + - orion + - hercules + - hera + - gaea diff --git a/env/HERA.env b/env/HERA.env index 80cd7cddaf..259461b1ac 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -153,10 +153,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF=${APRUN_default} elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 45fd607aa5..06ae2c1a63 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -149,10 +149,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index cea24fb26b..c67c16f929 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -126,10 +126,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default}" elif [[ "${step}" = "atmanlfv3inc" ]]; then diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 38dc3049f9..2a88f89eab 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -1,6 +1,13 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanalletkf" -c "base ocnanal marineanalletkf" + +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/${jobid}" +# Create the directory to hold ensemble perturbations +export DATAens="${DATAjob}/ensdata" +if [[ ! -d "${DATAens}" ]]; then mkdir -p "${DATAens}"; fi + +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlletkf" -c "base marineanl marineanlletkf" ############################################## # Set variables used in the script @@ -11,12 +18,18 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} +export GDUMP="gdas" +export GDUMP_ENS="enkf${GDUMP}" +export OPREFIX="${RUN}.t${cyc}z." -YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ - COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL +RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ + COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL \ + COMOUT_OCEAN_LETKF:COM_OCEAN_LETKF_TMPL \ + COMOUT_ICE_LETKF:COM_ICE_LETKF_TMPL ############################################## # Begin JOB SPECIFIC work @@ -25,7 +38,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL ############################################################### # Run relevant script -EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exgdas_global_marine_analysis_letkf.py} +EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exglobal_marine_analysis_letkf.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/marineanalletkf.sh b/jobs/rocoto/marineanlletkf.sh similarity index 95% rename from jobs/rocoto/marineanalletkf.sh rename to jobs/rocoto/marineanlletkf.sh index f2bfb9f70c..d4333461f3 100755 --- a/jobs/rocoto/marineanalletkf.sh +++ b/jobs/rocoto/marineanlletkf.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="marineanalletkf" +export job="marineanlletkf" export jobid="${job}.$$" ############################################################### diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 61d592561d..d949edb33a 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -82,12 +82,14 @@ declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history' declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart' declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input' declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' +declare -rx COM_OCEAN_LETKF_TMPL=${COM_BASE}'/analysis/ocean/letkf' declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean' declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2' declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' +declare -rx COM_ICE_LETKF_TMPL=${COM_BASE}'/analysis/ice/letkf' declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice' declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input' declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history' diff --git a/parm/config/gfs/config.marineanalletkf b/parm/config/gfs/config.marineanalletkf deleted file mode 100644 index fde3433a13..0000000000 --- a/parm/config/gfs/config.marineanalletkf +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -########## config.marineanalletkf ########## -# Ocn Analysis specific - -echo "BEGIN: config.marineanalletkf" - -# Get task specific resources -. "${EXPDIR}/config.resources" marineanalletkf - -export MARINE_LETKF_EXEC="${JEDI_BIN}/gdas.x" -export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" -export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml.j2" - -export GRIDGEN_EXEC="${JEDI_BIN}/gdas_soca_gridgen.x" -export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" - -echo "END: config.marineanalletkf" diff --git a/parm/config/gfs/config.marineanlletkf b/parm/config/gfs/config.marineanlletkf new file mode 100644 index 0000000000..8b84af4eaa --- /dev/null +++ b/parm/config/gfs/config.marineanlletkf @@ -0,0 +1,20 @@ +#!/bin/bash + +########## config.marineanlletkf ########## +# Ocn Analysis specific + +echo "BEGIN: config.marineanlletkf" + +# Get task specific resources +. "${EXPDIR}/config.resources" marineanlletkf + +export MARINE_LETKF_EXEC="${EXECgfs}/gdas.x" +export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" +export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml.j2" +export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j2" + +export GRIDGEN_EXEC="${EXECgfs}/gdas_soca_gridgen.x" +export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" +export DIST_HALO_SIZE=500000 + +echo "END: config.marineanlletkf" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 79dbb487db..14e6f0d7fb 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -601,7 +601,7 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; - "marineanalletkf") + "marineanlletkf") ntasks=16 case ${OCNRES} in "025") diff --git a/parm/stage/ocean_ens_perturbations.yaml.j2 b/parm/stage/ocean_ens_perturbations.yaml.j2 index fede3816a7..586b9f66cb 100644 --- a/parm/stage/ocean_ens_perturbations.yaml.j2 +++ b/parm/stage/ocean_ens_perturbations.yaml.j2 @@ -9,5 +9,5 @@ ocean_ens_perturbation: {% for mem in range(first_mem + 1, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc"] {% endfor %} # mem loop diff --git a/scripts/exgdas_global_marine_analysis_letkf.py b/scripts/exglobal_marine_analysis_letkf.py similarity index 100% rename from scripts/exgdas_global_marine_analysis_letkf.py rename to scripts/exglobal_marine_analysis_letkf.py diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 288b251aa8..25b2e28d75 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -466,12 +466,13 @@ MOM6_postdet() { fi # GEFS perturbations - # TODO if [[ $RUN} == "gefs" ]] block maybe be needed - # to ensure it does not interfere with the GFS when ensemble is updated in the GFS - if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then - ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ - || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) - fi + if [[ "${RUN}" == "gefs" ]]; then + # to ensure it does not interfere with the GFS + if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then + ${NCP} "${COMIN_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" \ + || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) + fi + fi # if [[ "${RUN}" == "gefs" ]]; then fi # if [[ "${RERUN}" == "NO" ]]; then # Link output files diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 93329f05ac..a4a5b4f144 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -318,7 +318,7 @@ def finalize(self: Task) -> None: FileHandler({'copy': diagb_list}).sync() # Copy the ensemble perturbation diagnostics to the ROTDIR - if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: window_middle_iso = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') weight_list = [] src = os.path.join(self.task_config.DATA, f"ocn.ens_weights.incr.{window_middle_iso}.nc") diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 36c26d594b..54d40f8d66 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import f90nml +import pygfs.utils.marine_da_utils as mdau from logging import getLogger import os from pygfs.task.analysis import Analysis from typing import Dict from wxflow import (AttrDict, + Executable, FileHandler, logit, parse_j2yaml, @@ -41,6 +43,8 @@ def __init__(self, config: Dict) -> None: 'soca', 'localensembleda', _letkf_yaml_file] + # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert + _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) self.task_config.WINDOW_MIDDLE = self.task_config.current_cycle self.task_config.WINDOW_BEGIN = self.task_config.current_cycle - _half_assim_freq @@ -49,6 +53,7 @@ def __init__(self, config: Dict) -> None: self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') + self.task_config.ENSPERT_RELPATH = _enspert_relpath @logit(logger) def initialize(self): @@ -64,26 +69,50 @@ def initialize(self): logger.info("initialize") # make directories and stage ensemble background files - ensbkgconf = AttrDict() - keys = ['previous_cycle', 'current_cycle', 'DATA', 'NMEM_ENS', - 'PARMgfs', 'ROTDIR', 'COM_OCEAN_HISTORY_TMPL', 'COM_ICE_HISTORY_TMPL'] - for key in keys: - ensbkgconf[key] = self.task_config[key] - ensbkgconf.RUN = 'enkfgdas' - soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, ensbkgconf) - FileHandler(soca_ens_bkg_stage_list).sync() soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() - letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config) + stageconf = AttrDict() + keys = ['current_cycle', + 'previous_cycle', + 'COM_ICE_LETKF_TMPL', + 'COM_OCEAN_LETKF_TMPL', + 'COM_ICE_HISTORY_TMPL', + 'COM_OCEAN_HISTORY_TMPL', + 'COMIN_OCEAN_HISTORY_PREV', + 'COMIN_ICE_HISTORY_PREV', + 'COMOUT_ICE_LETKF', + 'COMOUT_OCEAN_LETKF', + 'DATA', + 'ENSPERT_RELPATH', + 'GDUMP_ENS', + 'NMEM_ENS', + 'OPREFIX', + 'PARMgfs', + 'ROTDIR', + 'RUN', + 'WINDOW_BEGIN', + 'WINDOW_MIDDLE'] + for key in keys: + stageconf[key] = self.task_config[key] + + # stage ensemble background files + soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconf) + FileHandler(soca_ens_bkg_stage_list).sync() + + # stage letkf-specific files + letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) FileHandler(letkf_stage_list).sync() - obs_list = parse_j2yaml(self.task_config.OBS_YAML, self.task_config) + obs_list = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config) # get the list of observations obs_files = [] for ob in obs_list['observers']: obs_name = ob['obs space']['name'].lower() - obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc" + # TODO(AFE) - this should be removed when the obs config yamls are jinjafied + if 'distribution' not in ob['obs space']: + ob['obs space']['distribution'] = {'name': 'Halo', 'halo size': self.task_config['DIST_HALO_SIZE']} + obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc4" obs_files.append((obs_filename, ob)) obs_files_to_copy = [] @@ -102,12 +131,7 @@ def initialize(self): FileHandler({'copy': obs_files_to_copy}).sync() # make the letkf.yaml - letkfconf = AttrDict() - keys = ['WINDOW_BEGIN', 'WINDOW_MIDDLE', 'RUN', 'gcyc', 'NMEM_ENS'] - for key in keys: - letkfconf[key] = self.task_config[key] - letkfconf.RUN = 'enkfgdas' - letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, letkfconf) + letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf) letkf_yaml.observations.observers = obs_to_use letkf_yaml.save(self.task_config.letkf_yaml_file) @@ -133,6 +157,18 @@ def run(self): logger.info("run") + exec_cmd_gridgen = Executable(self.task_config.APRUN_MARINEANLLETKF) + exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_EXEC) + exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_YAML) + + mdau.run(exec_cmd_gridgen) + + exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANLLETKF) + for letkf_exec_arg in self.task_config.letkf_exec_args: + exec_cmd_letkf.add_default_arg(letkf_exec_arg) + + mdau.run(exec_cmd_letkf) + @logit(logger) def finalize(self): """Method finalize for ocean and sea ice LETKF task @@ -145,3 +181,11 @@ def finalize(self): """ logger.info("finalize") + + letkfsaveconf = AttrDict() + keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN', 'GDUMP_ENS', + 'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL'] + for key in keys: + letkfsaveconf[key] = self.task_config[key] + letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf) + FileHandler(letkf_save_list).sync() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index da78166ede..f92bf95fba 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -46,7 +46,7 @@ def _get_app_configs(self): if self.do_jediocnvar: configs += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: - configs += ['ocnanalecen'] + configs += ['marineanlletkf', 'ocnanalecen'] configs += ['marineanlchkpt', 'marineanlfinal'] if self.do_vrfy_oceanda: configs += ['ocnanalvrfy'] @@ -148,7 +148,7 @@ def get_task_names(self): if self.do_jediocnvar: gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: - gdas_gfs_common_tasks_before_fcst += ['ocnanalecen'] + gdas_gfs_common_tasks_before_fcst += ['marineanlletkf', 'ocnanalecen'] gdas_gfs_common_tasks_before_fcst += ['marineanlchkpt', 'marineanlfinal'] if self.do_vrfy_oceanda: gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 82dfb9f1d4..d3bb68a6b8 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -664,6 +664,32 @@ def prepoceanobs(self): return task + def marineanlletkf(self): + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'enkfgdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('marineanlletkf') + task_name = f'{self.run}_marineanlletkf' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlletkf.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def marinebmat(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 92ceea73aa..b989def13f 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -15,7 +15,7 @@ class Tasks: 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', - 'marineanlinit', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', + 'marineanlinit', 'marineanlletkf', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', From f4e380ac4c024d6778c333babbeba73601360d07 Mon Sep 17 00:00:00 2001 From: mingshichen-noaa <48537176+mingshichen-noaa@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:25:45 -0400 Subject: [PATCH 21/21] Update global jdas enkf diag job with COMIN/COMOUT for COM prefix (#2959) NCO has requested that each COM variable specify whether it is an input or an output. This completes that process for the global jdas enkf diagnostics job. Refs https://github.com/NOAA-EMC/global-workflow/issues/2451 --- jobs/JGDAS_ATMOS_ANALYSIS_DIAG | 5 ++-- jobs/JGDAS_ENKF_DIAG | 48 ++++++++++++++++++---------------- scripts/exglobal_diag.sh | 10 +++---- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG index a1e0c9f1d5..c47bd4a47b 100755 --- a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG +++ b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG @@ -27,8 +27,9 @@ export OPREFIX="${RUN/enkf}.t${cyc}z." export GPREFIX="${GDUMP}.t${gcyc}z." export APREFIX="${RUN}.t${cyc}z." -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS -mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}" +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL +mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGDAS_ENKF_DIAG b/jobs/JGDAS_ENKF_DIAG index cc8c933cc8..3daa8bfb73 100755 --- a/jobs/JGDAS_ENKF_DIAG +++ b/jobs/JGDAS_ENKF_DIAG @@ -30,56 +30,58 @@ export APREFIX="${RUN}.t${cyc}z." export GPREFIX="${GDUMP_ENS}.t${gcyc}z." GPREFIX_DET="${GDUMP}.t${gcyc}z." -RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS -MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS +RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL +MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_OBS_PREV:COM_OBS_TMPL \ - COM_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL + COMIN_OBS_PREV:COM_OBS_TMPL \ + COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL MEMDIR="ensstat" RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL + COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL -export ATMGES_ENSMEAN="${COM_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.ensmean.nc" +export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.ensmean.nc" if [ ! -f ${ATMGES_ENSMEAN} ]; then echo "FATAL ERROR: FILE MISSING: ATMGES_ENSMEAN = ${ATMGES_ENSMEAN}" exit 1 fi # Link observational data -export PREPQC="${COM_OBS}/${OPREFIX}prepbufr" +export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr" if [[ ! -f ${PREPQC} ]]; then echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING" fi -export TCVITL="${COM_OBS}/${OPREFIX}syndata.tcvitals.tm00" +export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00" if [[ ${DONST} = "YES" ]]; then - export NSSTBF="${COM_OBS}/${OPREFIX}nsstbufr" + export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr" fi -export PREPQCPF="${COM_OBS}/${OPREFIX}prepbufr.acft_profiles" +export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles" # Guess Bias correction coefficients related to control -export GBIAS=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias -export GBIASPC=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc -export GBIASAIR=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air -export GRADSTAT=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat +export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias +export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc +export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air +export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat # Bias correction coefficients related to ensemble mean -export ABIAS="${COM_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean" -export ABIASPC="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean" -export ABIASAIR="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean" -export ABIASe="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean" +export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean" +export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean" +export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean" +export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean" # Diagnostics related to ensemble mean -export GSISTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean" -export CNVSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean" -export OZNSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean" -export RADSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean" +export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean" +export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean" +export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean" +export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean" # Select observations based on ensemble mean export RUN_SELECT="YES" export USE_SELECT="NO" -export SELECT_OBS="${COM_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" +export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" export DIAG_SUFFIX="_ensmean" export DIAG_COMPRESS="NO" diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index ed9bef05df..46a6e9863c 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -49,10 +49,10 @@ SENDDBN=${SENDDBN:-"NO"} # Analysis files export APREFIX=${APREFIX:-""} -RADSTAT=${RADSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}radstat} -PCPSTAT=${PCPSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}pcpstat} -CNVSTAT=${CNVSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}cnvstat} -OZNSTAT=${OZNSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}oznstat} +RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat} +PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} +CNVSTAT=${CNVSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat} +OZNSTAT=${OZNSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat} # Remove stat file if file already exists [[ -s $RADSTAT ]] && rm -f $RADSTAT @@ -74,7 +74,7 @@ nm="" if [ $CFP_MP = "YES" ]; then nm=0 fi -DIAG_DIR=${DIAG_DIR:-${COM_ATMOS_ANALYSIS}/gsidiags} +DIAG_DIR=${DIAG_DIR:-${COMOUT_ATMOS_ANALYSIS}/gsidiags} REMOVE_DIAG_DIR=${REMOVE_DIAG_DIR:-"NO"} # Set script / GSI control parameters