Merge remote-tracking branch 'EMC/develop' into feature/addmoreprods

.github/workflows/pw_aws_centos.yaml → .github/workflows/pw_aws_ci.yaml

@@ -1,7 +1,4 @@
-name: gw-ci-aws-centos
-
-on: [workflow_dispatch]
-
+name: gw-ci-aws
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow
 # 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
@@ -10,33 +7,73 @@ on: [workflow_dispatch]
 # ├── HOMEgfs
 # └── RUNTESTS
 #     ├── COMROT
-#     │   └── ${pslot}
+#     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'Pull Request Number (use 0 for non-PR)'
+        required: true
+        default: '0'
+      os:
+        description: 'Operating System'
+        required: true
+        type: choice
+        options:
+          - rocky
+          - centos
+
 env:
   TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
   MACHINE_ID: noaacloud
 
 jobs:
+  fetch-branch:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUBTOKEN }}
+    outputs:
+      branch: ${{ steps.get-branch.outputs.branch }}
+    steps:
+    - name: Fetch branch name for PR
+      id: get-branch
+      run: |
+        pr_number=${{ github.event.inputs.pr_number }}
+        repo=${{ github.repository }}
+        if [ "$pr_number" -eq "0" ]; then
+          branch=${{ github.event.inputs.ref }}
+        else
+          branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName')
+        fi
+        echo "::set-output name=branch::$branch"
+
   checkout:
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: fetch-branch
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     timeout-minutes: 600
-
     steps:
-
     - name: Checkout global-workflow
       uses: actions/checkout@v4
       with:
         path: ${{ github.run_id }}/HOMEgfs
         submodules: 'recursive'
-        ref: ${{ github.event.pull_request.head.ref }}
+        ref: ${{ needs.fetch-branch.outputs.branch }}
 
   build-link:      
-    runs-on: [self-hosted, aws, parallelworks, centos]
     needs: checkout
-
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     steps:
-
     - name: Build components
       run: |
         cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
@@ -48,12 +85,15 @@ jobs:
         ./link_workflow.sh
 
   create-experiments:
-    needs: checkout
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: build-link
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     strategy:
       matrix:
         case: ["C48_ATM"]
-
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
@@ -68,9 +108,12 @@ jobs:
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     strategy:
-      max-parallel: 2
       matrix:
         case: ["C48_ATM"]
     steps:
@@ -81,9 +124,13 @@ jobs:
 
   clean-up:
     needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     steps:
-      - name: Clean-up
+      - name: Clean up workspace
         run: |
-          cd ${{ github.workspace }}
-          rm -rf ${{ github.run_id }}
+          echo "Cleaning up workspace"
+          rm -rf ${{ env.TEST_DIR }}

ci/Jenkinsfile

@@ -5,7 +5,7 @@ def cases = ''
 def GH = 'none'
 // Location of the custom workspaces for each machine in the CI system.  They are persitent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
 def repo_url = '[email protected]:NOAA-EMC/global-workflow.git'
 def STATUS = 'Passed'
 
@@ -101,9 +101,10 @@ pipeline {
                 stages {
                     stage('Building') {
                         steps {
-                           catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                          catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                             script {
                                 def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system
+                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                 sh(script: "mkdir -p ${HOMEgfs}")
                                 ws(HOMEgfs) {
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
@@ -172,9 +173,10 @@ pipeline {
                                     }
                                     if (system == 'gfs') {
                                         cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                                        echo "Cases to run: ${cases}"
                                     }
                                }
-                           }
+                            }
                           }
                         }
                     }
@@ -192,32 +194,34 @@ pipeline {
                     def parallelStages = cases.collectEntries { caseName ->
                         ["${caseName}": {
                             stage("Create ${caseName}") {
-                                    catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-                                        script {
-                                                sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
-                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
-                                                system = yaml_case.experiment.system
-                                                def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
-                                                env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
-                                                try {
-                                                    error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
-                                                } catch (Exception error_create) {
-                                                    sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
-                                                    error("Case ${caseName} failed to create experment directory")
-                                                }
+                                script {
+                                        sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
+                                        def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                        def build_system = yaml_case.experiment.system
+                                        def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}"   // local HOMEgfs is used to populate the XML on per system basis
+                                        env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
+                                        env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
+                                        try {
+                                              error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
+                                        } catch (Exception error_create) {
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
+                                            error("Case ${caseName} failed to create experment directory")
                                         }
-                                }
+                                     }
                             }
 
                             stage("Running ${caseName}") {
                                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                                         script {
                                                 HOMEgfs = "${CUSTOM_WORKSPACE}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
+                                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                                 def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
+                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                                def build_system = yaml_case.experiment.system
                                                 try {
-                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {
@@ -268,11 +272,11 @@ pipeline {
             }
         }
 
-
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
             steps {
                 script {
+                    env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                     sh(script: """
                         labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name')
                         for label in \$labels; do

ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -18,6 +18,7 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml
 
 skip_ci_on_hosts:
+  - wcoss2
   - orion
   - gaea
   - hercules

ci/scripts/utils/launch_java_agent.sh

@@ -106,7 +106,7 @@ export GH="${HOME}/bin/gh"
 [[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin"
 ${GH} --version
 
-check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
+check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
 if [[ "${check_mark}" != "✓" ]]; then
   echo "gh not authenticating with emcbot token"
   exit 1

ush/python/pygfs/task/marine_analysis.py

@@ -210,6 +210,7 @@ def _prep_variational_yaml(self: Task) -> None:
         envconfig_jcb['cyc'] = os.getenv('cyc')
         envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER
         envconfig_jcb['obs_list'] = ['adt_rads_all']
+        envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES))
 
         # Write obs_list_short
         save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml')

ush/python/pygfs/utils/marine_da_utils.py

@@ -166,3 +166,36 @@ def clean_empty_obsspaces(config, target, app='var'):
 
     # save cleaned yaml
     save_as_yaml(config, target)
+
+
+@logit(logger)
+def get_mom6_levels(ocnres: str) -> int:
+    """
+    Temporary function that returns the number of vertical levels in MOM6 given the horizontal resolution.
+    This is requiered by the diffusion saber block that now makes use of oops::util::FieldSetHelpers::writeFieldSet
+    and requires the number of levels in the configuration. I have been told this will be changed in the future.
+
+    Parameters
+    -----------
+    ocnres: str
+        Input resolution for ocean in str format. e.g. '500', '100', '050', '025'
+
+   Returns
+   -------
+   nlev: int
+       number of levels in the ocean model given an input resolution
+    """
+
+    # Currently implemented resolutions
+    ocnres_to_nlev = {
+        '500': 25,
+        '100': 75,
+        '050': 75,
+        '025': 75
+    }
+    try:
+        nlev = ocnres_to_nlev.get(ocnres)
+    except KeyError:
+        raise KeyError("FATAL ERROR: Invalid ocnres value. Aborting.")
+
+    return nlev

versions/fix.ver

@@ -8,7 +8,7 @@ export cice_ver=20240416
 export cpl_ver=20230526
 export datm_ver=20220805
 export gdas_crtm_ver=20220805
-export gdas_fv3jedi_ver=20220805
+export gdas_fv3jedi_ver=20241022
 export gdas_soca_ver=20240802
 export gdas_gsibec_ver=20240416
 export gdas_obs_ver=20240213

workflow/generate_workflows.sh

@@ -71,6 +71,9 @@ function _usage() {
        If this option is not specified, then the existing email address in
        the crontab will be preserved.
 
+    -t Add a 'tag' to the end of the case names in the pslots to distinguish
+       pslots between multiple sets of tests.
+
     -v Verbose mode.  Prints output of all commands to stdout.
 
     -V Very verbose mode.  Passes -v to all commands and prints to stdout.
@@ -101,6 +104,7 @@ _hpc_account=""
 _set_account=false
 _update_cron=false
 _email=""
+_tag=""
 _set_email=false
 _verbose=false
 _very_verbose=false
@@ -111,7 +115,7 @@ _runtests="${RUNTESTS:-${_runtests:-}}"
 _nonflag_option_count=0
 
 while [[ $# -gt 0 && "$1" != "--" ]]; do
-   while getopts ":H:bB:uy:Y:GESA:ce:vVdh" option; do
+   while getopts ":H:bB:uy:Y:GESA:ce:t:vVdh" option; do
       case "${option}" in
         H)
            HOMEgfs="${OPTARG}"
@@ -138,6 +142,7 @@ while [[ $# -gt 0 && "$1" != "--" ]]; do
         S) _run_all_sfs=true ;;
         c) _update_cron=true ;;
         e) _email="${OPTARG}" && _set_email=true ;;
+        t) _tag="_${OPTARG}" ;;
         v) _verbose=true ;;
         V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;;
         d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;;
@@ -454,11 +459,12 @@ echo "Running create_experiment.py for ${#_yaml_list[@]} cases"
 [[ "${_verbose}" == true ]] && printf "Selected cases: %s\n\n" "${_yaml_list[*]}"
 for _case in "${_yaml_list[@]}"; do
    [[ "${_verbose}" == false ]] && echo "${_case}"
+   _pslot="${_case}${_tag}"
    _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite"
    if [[ "${_verbose}" == true ]]; then
-      pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd}
+      pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd}
    else
-      if ! pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then
+      if ! pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then
          _output=$(cat stdout stderr)
          _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status.  Output:"
          _message="${_message}"$'\n'"${_output}"
@@ -471,7 +477,7 @@ for _case in "${_yaml_list[@]}"; do
       fi
       rm -f stdout stderr
    fi
-   grep "${_case}" "${_runtests}/EXPDIR/${_case}/${_case}.crontab" >> tests.cron
+   grep "${_pslot}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab" >> tests.cron
 done
 echo