Merge branch 'develop' into SFS

.github/workflows/pw_aws_ci.yaml

@@ -15,7 +15,7 @@ on:
   workflow_dispatch:
     inputs:
       pr_number:
-        description: 'Pull Request Number (use 0 for non-PR)'
+        description: 'PR Number (use 0 for non-PR)'
         required: true
         default: '0'
       os:
@@ -31,24 +31,60 @@ env:
   MACHINE_ID: noaacloud
 
 jobs:
+
+  run-start-clusters:
+    runs-on: ubuntu-latest
+    env:
+      PW_PLATFORM_HOST: noaa.parallel.works
+    steps:
+    - name: Checkout pw-cluster-automation repository
+      uses: actions/checkout@v4
+      with:
+        repository: TerrenceMcGuinness-NOAA/pw-cluster-automation
+        path: pw-cluster-automation
+        ref: pw_cluster_noaa
+
+    - name: Run startClusters
+      run: |
+        mkdir -p ~/.ssh
+        echo "${{ secrets.ID_RSA_AWS }}" > ~/.ssh/id_rsa
+        echo "${{ secrets.PW_API_KEY }}" > ~/.ssh/pw_api.key
+        chmod 700 ~/.ssh
+        chmod 600 ~/.ssh/id_rsa
+        chmod 600 ~/.ssh/pw_api.key
+        if [ "${{ github.event.inputs.os }}" == "rocky" ]; then
+          clustername="globalworkflowciplatformrocky8"
+        elif [ "${{ github.event.inputs.os }}" == "centos" ]; then
+          clustername="awsemctmcgc7i48xlargeciplatform"
+        fi
+        python3 pw-cluster-automation/startClusters.py $clustername
+
   fetch-branch:
+    needs: run-start-clusters
     runs-on: ubuntu-latest
     env:
       GH_TOKEN: ${{ secrets.GITHUBTOKEN }}
     outputs:
       branch: ${{ steps.get-branch.outputs.branch }}
+      repo: ${{ steps.get-branch.outputs.repo }}
     steps:
-    - name: Fetch branch name for PR
+    - name: Fetch branch name and repo for PR
       id: get-branch
       run: |
         pr_number=${{ github.event.inputs.pr_number }}
-        repo=${{ github.repository }}
         if [ "$pr_number" -eq "0" ]; then
           branch=${{ github.event.inputs.ref }}
+          repo=${{ github.repository }}
         else
           branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName')
+          repo_owner=$(gh pr view $pr_number --repo $repo --json headRepositoryOwner --jq '.headRepositoryOwner.login')
+          repo_name=$(gh pr view $pr_number --repo $repo --json headRepository --jq '.headRepository.name')
+          repo="$repo_owner/$repo_name"
         fi
-        echo "::set-output name=branch::$branch"
+        {
+          echo "branch=$branch"
+          echo "repo=$repo"
+        } >> $GITHUB_OUTPUT
 
   checkout:
     needs: fetch-branch
@@ -64,6 +100,7 @@ jobs:
       with:
         path: ${{ github.run_id }}/HOMEgfs
         submodules: 'recursive'
+        repository: ${{ needs.fetch-branch.outputs.repo }}
         ref: ${{ needs.fetch-branch.outputs.branch }}
 
   build-link:      

.gitignore

@@ -172,11 +172,6 @@ ush/bufr2ioda_insitu*
 versions/build.ver
 versions/run.ver
 
-# wxflow checkout and symlinks
-ush/python/wxflow
-workflow/wxflow
-ci/scripts/wxflow
-
 # jcb checkout and symlinks
 ush/python/jcb
 workflow/jcb

ci/cases/pr/C48mx500_hybAOWCDA.yaml

@@ -0,0 +1,26 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 48
+  resdetocean: 5.0
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610
+  idate: 2021032412
+  edate: 2021032418
+  nens: 3
+  interval: 0
+  start: warm
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - wcoss2
+  - orion
+  - hercules
+  - hera
+  - gaea

ci/scripts/wxflow

@@ -0,0 +1 @@
+../../sorc/wxflow/src/wxflow

env/HERA.env

@@ -9,7 +9,7 @@ fi
 
 step=$1
 
-export launcher="srun -l --export=ALL"
+export launcher="srun -l --export=ALL --hint=nomultithread"
 export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 #export POSTAMBLE_CMD='report-mem'
@@ -50,7 +50,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export POE="NO"
     export BACK="NO"
     export sys_tp="HERA"
-    export launcher_PREP="srun"
+    export launcher_PREP="srun --hint=nomultithread"
 
 elif [[ "${step}" = "prepsnowobs" ]]; then
 
@@ -153,10 +153,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then
     export NTHREADS_OCNANALECEN=${NTHREADSmax}
     export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}"
 
-elif [[ "${step}" = "marineanalletkf" ]]; then
+elif [[ "${step}" = "marineanlletkf" ]]; then
 
-    export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
-    export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"
+    export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
+    export APRUN_MARINEANLLETKF=${APRUN_default}
 
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 

env/HERCULES.env

@@ -9,7 +9,7 @@ fi
 
 step=$1
 
-export launcher="srun -l --export=ALL"
+export launcher="srun -l --export=ALL --hint=nomultithread"
 export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
@@ -48,7 +48,7 @@ case ${step} in
     export POE="NO"
     export BACK=${BACK:-"YES"}
     export sys_tp="HERCULES"
-    export launcher_PREP="srun"
+    export launcher_PREP="srun --hint=nomultithread"
  ;;
  "prepsnowobs")
 

env/ORION.env

@@ -9,7 +9,7 @@ fi
 
 step=$1
 
-export launcher="srun -l --export=ALL"
+export launcher="srun -l --export=ALL --hint=nomultithread"
 export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
@@ -45,7 +45,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export POE="NO"
     export BACK=${BACK:-"YES"}
     export sys_tp="ORION"
-    export launcher_PREP="srun"
+    export launcher_PREP="srun --hint=nomultithread"
 
 elif [[ "${step}" = "prepsnowobs" ]]; then
 
@@ -149,10 +149,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then
     export NTHREADS_OCNANALECEN=${NTHREADSmax}
     export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}"
 
-elif [[ "${step}" = "marineanalletkf" ]]; then
+elif [[ "${step}" = "marineanlletkf" ]]; then
 
-    export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
-    export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"
+    export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
+    export APRUN_MARINEANLLETKF="${APRUN_default}"
 
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 

env/WCOSS2.env

@@ -126,10 +126,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then
     export NTHREADS_OCNANALECEN=${NTHREADSmax}
     export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}"
 
-elif [[ "${step}" = "marineanalletkf" ]]; then
+elif [[ "${step}" = "marineanlletkf" ]]; then
 
-    export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
-    export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"
+    export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
+    export APRUN_MARINEANLLETKF="${APRUN_default}"
 
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 

jobs/JGDAS_ATMOS_ANALYSIS_DIAG

@@ -27,8 +27,9 @@ export OPREFIX="${RUN/enkf}.t${cyc}z."
 export GPREFIX="${GDUMP}.t${gcyc}z."
 export APREFIX="${RUN}.t${cyc}z."
 
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS
-mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}"
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+	COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}"
 
 ###############################################################
 # Run relevant script

jobs/JGDAS_ENKF_DIAG

@@ -30,56 +30,58 @@ export APREFIX="${RUN}.t${cyc}z."
 export GPREFIX="${GDUMP_ENS}.t${gcyc}z."
 GPREFIX_DET="${GDUMP}.t${gcyc}z."
 
-RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS
-MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS
+RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+	COMIN_OBS:COM_OBS_TMPL
+MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+	COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_OBS_PREV:COM_OBS_TMPL \
-    COM_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL
+    COMIN_OBS_PREV:COM_OBS_TMPL \
+    COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL
 
 MEMDIR="ensstat" RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
+    COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
 
 
-export ATMGES_ENSMEAN="${COM_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.ensmean.nc"
+export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.ensmean.nc"
 if [ ! -f ${ATMGES_ENSMEAN} ]; then
     echo "FATAL ERROR: FILE MISSING: ATMGES_ENSMEAN = ${ATMGES_ENSMEAN}"
     exit 1
 fi
 
 # Link observational data
-export PREPQC="${COM_OBS}/${OPREFIX}prepbufr"
+export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr"
 if [[ ! -f ${PREPQC} ]]; then
     echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING"
 fi
-export TCVITL="${COM_OBS}/${OPREFIX}syndata.tcvitals.tm00"
+export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00"
 if [[ ${DONST} = "YES" ]]; then
-   export NSSTBF="${COM_OBS}/${OPREFIX}nsstbufr"
+   export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr"
 fi
-export PREPQCPF="${COM_OBS}/${OPREFIX}prepbufr.acft_profiles"
+export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles"
 
 # Guess Bias correction coefficients related to control
-export GBIAS=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias
-export GBIASPC=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc
-export GBIASAIR=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air
-export GRADSTAT=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat
+export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias
+export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc
+export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air
+export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat
 
 # Bias correction coefficients related to ensemble mean
-export ABIAS="${COM_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean"
-export ABIASPC="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean"
-export ABIASAIR="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean"
-export ABIASe="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean"
+export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean"
+export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean"
+export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean"
+export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean"
 
 # Diagnostics related to ensemble mean
-export GSISTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean"
-export CNVSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean"
-export OZNSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean"
-export RADSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean"
+export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean"
+export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean"
+export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean"
+export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean"
 
 # Select observations based on ensemble mean
 export RUN_SELECT="YES"
 export USE_SELECT="NO"
-export SELECT_OBS="${COM_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean"
+export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean"
 
 export DIAG_SUFFIX="_ensmean"
 export DIAG_COMPRESS="NO"

jobs/JGLOBAL_MARINE_ANALYSIS_LETKF

@@ -1,6 +1,13 @@
 #!/bin/bash
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanalletkf" -c "base ocnanal marineanalletkf"
+
+export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}"
+export DATA="${DATAjob}/${jobid}"
+# Create the directory to hold ensemble perturbations
+export DATAens="${DATAjob}/ensdata"
+if [[ ! -d "${DATAens}" ]]; then mkdir -p "${DATAens}"; fi
+
+source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlletkf" -c "base marineanl marineanlletkf"
 
 ##############################################
 # Set variables used in the script
@@ -11,12 +18,18 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
 
 gPDY=${GDATE:0:8}
 gcyc=${GDATE:8:2}
+export GDUMP="gdas"
+export GDUMP_ENS="enkf${GDUMP}"
+export OPREFIX="${RUN}.t${cyc}z."
 
-YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-   COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \
-   COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL
+RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+  COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \
+  COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL
 
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL 
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+  COMIN_OBS:COM_OBS_TMPL \
+  COMOUT_OCEAN_LETKF:COM_OCEAN_LETKF_TMPL \
+  COMOUT_ICE_LETKF:COM_ICE_LETKF_TMPL
 
 ##############################################
 # Begin JOB SPECIFIC work
@@ -25,7 +38,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exgdas_global_marine_analysis_letkf.py}
+EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exglobal_marine_analysis_letkf.py}
 ${EXSCRIPT}
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"

jobs/rocoto/marineanalletkf.sh → jobs/rocoto/marineanlletkf.sh

@@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh"
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"
 
-export job="marineanalletkf"
+export job="marineanlletkf"
 export jobid="${job}.$$"
 
 ###############################################################

parm/config/gefs/config.efcs

@@ -26,18 +26,6 @@ source "${EXPDIR}/config.ufs" ${string}
 # Get task specific resources
 source "${EXPDIR}/config.resources" efcs
 
-# nggps_diag_nml
-export FHOUT=${FHOUT_ENKF:-3}
-if [[ "${RUN}" == "enkfgfs" ]]; then
-    export FHOUT=${FHOUT_ENKF_GFS:-${FHOUT}}
-fi
-
-# model_configure
-export FHMAX=${FHMAX_ENKF:-9}
-if [[ "${RUN}" == "enkfgfs" ]]; then
-   export FHMAX=${FHMAX_ENKF_GFS:-${FHMAX}}
-fi
-
 # Stochastic physics parameters (only for ensemble forecasts)
 if [[ "${CASE}" == "C96" ]] ; then
     export DO_SKEB="NO" # SKEB turned off for C96
@@ -78,6 +66,6 @@ if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
 else
     export ODA_INCUPD="False"
 fi
-export restart_interval="${restart_interval_enkfgfs:-12}"
+export restart_interval="${restart_interval_gfs:-12}"
 
 echo "END: config.efcs"