diff --git a/.github/workflows/pw_aws_centos.yaml b/.github/workflows/pw_aws_ci.yaml similarity index 51% rename from .github/workflows/pw_aws_centos.yaml rename to .github/workflows/pw_aws_ci.yaml index 76258c3044..245e219dd4 100644 --- a/.github/workflows/pw_aws_centos.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -1,7 +1,4 @@ -name: gw-ci-aws-centos - -on: [workflow_dispatch] - +name: gw-ci-aws # TEST_DIR contains 2 directories; # 1. HOMEgfs: clone of the global-workflow # 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments @@ -10,33 +7,73 @@ on: [workflow_dispatch] # ├── HOMEgfs # └── RUNTESTS # ├── COMROT -# │   └── ${pslot} +# │ └── ${pslot} # └── EXPDIR # └── ${pslot} + +on: + workflow_dispatch: + inputs: + pr_number: + description: 'Pull Request Number (use 0 for non-PR)' + required: true + default: '0' + os: + description: 'Operating System' + required: true + type: choice + options: + - rocky + - centos + env: TEST_DIR: ${{ github.workspace }}/${{ github.run_id }} MACHINE_ID: noaacloud jobs: + fetch-branch: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUBTOKEN }} + outputs: + branch: ${{ steps.get-branch.outputs.branch }} + steps: + - name: Fetch branch name for PR + id: get-branch + run: | + pr_number=${{ github.event.inputs.pr_number }} + repo=${{ github.repository }} + if [ "$pr_number" -eq "0" ]; then + branch=${{ github.event.inputs.ref }} + else + branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') + fi + echo "::set-output name=branch::$branch" + checkout: - runs-on: [self-hosted, aws, parallelworks, centos] + needs: fetch-branch + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} timeout-minutes: 600 - steps: - - name: Checkout global-workflow uses: actions/checkout@v4 with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ needs.fetch-branch.outputs.branch }} build-link: - runs-on: [self-hosted, aws, parallelworks, centos] needs: checkout - + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} steps: - - name: Build components run: | cd ${{ env.TEST_DIR }}/HOMEgfs/sorc @@ -48,12 +85,15 @@ jobs: ./link_workflow.sh create-experiments: - needs: checkout - runs-on: [self-hosted, aws, parallelworks, centos] + needs: build-link + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} strategy: matrix: case: ["C48_ATM"] - steps: - name: Create Experiments ${{ matrix.case }} env: @@ -68,9 +108,12 @@ jobs: run-experiments: needs: create-experiments - runs-on: [self-hosted, aws, parallelworks, centos] + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} strategy: - max-parallel: 2 matrix: case: ["C48_ATM"] steps: @@ -81,9 +124,13 @@ jobs: clean-up: needs: run-experiments - runs-on: [self-hosted, aws, parallelworks, centos] + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} steps: - - name: Clean-up + - name: Clean up workspace run: | - cd ${{ github.workspace }} - rm -rf ${{ github.run_id }} + echo "Cleaning up workspace" + rm -rf ${{ env.TEST_DIR }} diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 2654adba29..6a2e064be0 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -5,7 +5,7 @@ def cases = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' @@ -101,9 +101,10 @@ pipeline { stages { stage('Building') { steps { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc sh(script: "mkdir -p ${HOMEgfs}") ws(HOMEgfs) { if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs @@ -172,9 +173,10 @@ pipeline { } if (system == 'gfs') { cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() + echo "Cases to run: ${cases}" } } - } + } } } } @@ -192,32 +194,34 @@ pipeline { def parallelStages = cases.collectEntries { caseName -> ["${caseName}": { stage("Create ${caseName}") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") - def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" - system = yaml_case.experiment.system - def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis - env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" - try { - error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() - } catch (Exception error_create) { - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) - error("Case ${caseName} failed to create experment directory") - } + script { + sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + def build_system = yaml_case.experiment.system + def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}" // local HOMEgfs is used to populate the XML on per system basis + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc + env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" + try { + error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() + } catch (Exception error_create) { + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) + error("Case ${caseName} failed to create experment directory") } - } + } } stage("Running ${caseName}") { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { script { HOMEgfs = "${CUSTOM_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + def build_system = yaml_case.experiment.system try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}") + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") } catch (Exception error_experment) { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") ws(CUSTOM_WORKSPACE) { @@ -268,11 +272,11 @@ pipeline { } } - stage( '5. FINALIZE' ) { agent { label NodeName[machine].toLowerCase() } steps { script { + env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc sh(script: """ labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name') for label in \$labels; do diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index be5ad32238..57d0989ae3 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -18,6 +18,7 @@ arguments: yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml skip_ci_on_hosts: + - wcoss2 - orion - gaea - hercules diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 183e671b9d..eb78d3b1ef 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -106,7 +106,7 @@ export GH="${HOME}/bin/gh" [[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin" ${GH} --version -check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true +check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true if [[ "${check_mark}" != "✓" ]]; then echo "gh not authenticating with emcbot token" exit 1 diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 55e895f1dc..764f58cebd 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 +Subproject commit 764f58cebdf64f3695d89538994a50183e5884d9 diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 4e4311b906..e7b7b5e948 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -210,6 +210,7 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['cyc'] = os.getenv('cyc') envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER envconfig_jcb['obs_list'] = ['adt_rads_all'] + envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES)) # Write obs_list_short save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index e1b2ac2d4d..50d9d84e86 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -166,3 +166,36 @@ def clean_empty_obsspaces(config, target, app='var'): # save cleaned yaml save_as_yaml(config, target) + + +@logit(logger) +def get_mom6_levels(ocnres: str) -> int: + """ + Temporary function that returns the number of vertical levels in MOM6 given the horizontal resolution. + This is requiered by the diffusion saber block that now makes use of oops::util::FieldSetHelpers::writeFieldSet + and requires the number of levels in the configuration. I have been told this will be changed in the future. + + Parameters + ----------- + ocnres: str + Input resolution for ocean in str format. e.g. '500', '100', '050', '025' + + Returns + ------- + nlev: int + number of levels in the ocean model given an input resolution + """ + + # Currently implemented resolutions + ocnres_to_nlev = { + '500': 25, + '100': 75, + '050': 75, + '025': 75 + } + try: + nlev = ocnres_to_nlev.get(ocnres) + except KeyError: + raise KeyError("FATAL ERROR: Invalid ocnres value. Aborting.") + + return nlev diff --git a/versions/fix.ver b/versions/fix.ver index 7c18bea081..b175791196 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -8,7 +8,7 @@ export cice_ver=20240416 export cpl_ver=20230526 export datm_ver=20220805 export gdas_crtm_ver=20220805 -export gdas_fv3jedi_ver=20220805 +export gdas_fv3jedi_ver=20241022 export gdas_soca_ver=20240802 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index c4f89eef6e..c2565140d3 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -71,6 +71,9 @@ function _usage() { If this option is not specified, then the existing email address in the crontab will be preserved. + -t Add a 'tag' to the end of the case names in the pslots to distinguish + pslots between multiple sets of tests. + -v Verbose mode. Prints output of all commands to stdout. -V Very verbose mode. Passes -v to all commands and prints to stdout. @@ -101,6 +104,7 @@ _hpc_account="" _set_account=false _update_cron=false _email="" +_tag="" _set_email=false _verbose=false _very_verbose=false @@ -111,7 +115,7 @@ _runtests="${RUNTESTS:-${_runtests:-}}" _nonflag_option_count=0 while [[ $# -gt 0 && "$1" != "--" ]]; do - while getopts ":H:bB:uy:Y:GESA:ce:vVdh" option; do + while getopts ":H:bB:uy:Y:GESA:ce:t:vVdh" option; do case "${option}" in H) HOMEgfs="${OPTARG}" @@ -138,6 +142,7 @@ while [[ $# -gt 0 && "$1" != "--" ]]; do S) _run_all_sfs=true ;; c) _update_cron=true ;; e) _email="${OPTARG}" && _set_email=true ;; + t) _tag="_${OPTARG}" ;; v) _verbose=true ;; V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;; d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;; @@ -454,11 +459,12 @@ echo "Running create_experiment.py for ${#_yaml_list[@]} cases" [[ "${_verbose}" == true ]] && printf "Selected cases: %s\n\n" "${_yaml_list[*]}" for _case in "${_yaml_list[@]}"; do [[ "${_verbose}" == false ]] && echo "${_case}" + _pslot="${_case}${_tag}" _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite" if [[ "${_verbose}" == true ]]; then - pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} + pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} else - if ! pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then + if ! pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then _output=$(cat stdout stderr) _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status. Output:" _message="${_message}"$'\n'"${_output}" @@ -471,7 +477,7 @@ for _case in "${_yaml_list[@]}"; do fi rm -f stdout stderr fi - grep "${_case}" "${_runtests}/EXPDIR/${_case}/${_case}.crontab" >> tests.cron + grep "${_pslot}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab" >> tests.cron done echo