Bring CDEPS inline capability to CMEPS (#2028)

Developed and enabled using CMEPS with inline CDEPS capability for UFS regional coupling.
Two HAFS regional moving nest atm_ocn_wav coupling regression tests were added to test this capability.
Note: This is a collaborative effort among the ESMF team, EMC/EIB, and EMC hurricane project team.

CDEPS-interface/cdeps_files.cmake

@@ -12,8 +12,10 @@ list(APPEND cdeps_share_files
   CDEPS/share/shr_assert.h
   CDEPS/share/shr_cal_mod.F90
   CDEPS/share/shr_const_mod.F90
+  CDEPS/share/shr_file_mod.F90
   CDEPS/share/shr_kind_mod.F90
   CDEPS/share/shr_log_mod.F90
+  CDEPS/share/shr_nl_mod.F90
   CDEPS/share/shr_orb_mod.F90
   CDEPS/share/shr_precip_mod.F90
   CDEPS/share/shr_strconvert_mod.F90

CMEPS-interface/CMakeLists.txt

@@ -81,6 +81,11 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_phases_post_rof_mod.F90
   CMEPS/mediator/med_phases_post_wav_mod.F90)
 
+if(CDEPS_INLINE)
+  list(APPEND _mediator_files CMEPS/mediator/med_phases_cdeps_mod.F90)
+  add_definitions(-DCDEPS_INLINE)
+endif()
+
 ###############################################################################
 ### Host model for CCPP
 ###############################################################################
@@ -184,6 +189,12 @@ if(OpenMP_Fortran_FOUND)
   target_link_libraries(cmeps PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 
+### Set dependency for CDEPS if inline capability is requested
+if(CDEPS_INLINE)
+  add_dependencies(cmeps cdeps::cdeps)
+  target_link_libraries(cmeps PUBLIC cdeps::cdeps)
+endif()
+
 ###############################################################################
 ### Install
 ###############################################################################

CMakeLists.txt

@@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL HAFS HAFSW HAFS-ALL NG-GODAS)
+list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -66,11 +66,13 @@ set(DISABLE_FMA     OFF CACHE BOOL "Disable Fused Multiply-Add instructions (wor
 set(INLINE_POST     ON  CACHE BOOL "Enable inline post")
 set(MULTI_GASES     OFF CACHE BOOL "Enable MULTI_GASES")
 set(MOVING_NEST     OFF CACHE BOOL "Enable moving nest code")
+set(REGIONAL_MOM6   OFF CACHE BOOL "Enable Regional MOM6")
 set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
 set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
 set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(CMEPS_AOFLUX    OFF CACHE BOOL "Enable atmosphere-ocean flux calculation in mediator")
 set(PDLIB           OFF CACHE BOOL "Enable Domain Decomposition in WW3 via PDLIB")
+set(CDEPS_INLINE    OFF CACHE BOOL "Enable CDEPS inline capability")
 
 set(CMAKE_Platform $ENV{CMAKE_Platform})
 if(CMAKE_Platform)
@@ -114,10 +116,12 @@ message("DEBUG ............ ${DEBUG}")
 message("INLINE_POST ...... ${INLINE_POST}")
 message("MULTI_GASES ...... ${MULTI_GASES}")
 message("MOVING_NEST ...... ${MOVING_NEST}")
+message("REGIONAL_MOM6..... ${REGIONAL_MOM6}")
 message("OPENMP ........... ${OPENMP}")
 message("PARALLEL_NETCDF .. ${PARALLEL_NETCDF}")
 message("JEDI_DRIVER ...... ${JEDI_DRIVER}")
 message("CMEPS_AOFLUX ..... ${CMEPS_AOFLUX}")
+message("CDEPS_INLINE ..... ${CDEPS_INLINE}")
 
 message("")
 
@@ -146,7 +150,7 @@ if(FMS)
   find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8)
   if(APP MATCHES "^(HAFSW)$")
     add_library(fms ALIAS FMS::fms_r4)
-  elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|NG-GODAS)$")
+  elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$")
     add_library(fms ALIAS FMS::fms_r8)
   endif()
   if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|HAFS|HAFS-ALL)$")

MOM6-interface/CMakeLists.txt

@@ -28,8 +28,14 @@ include("mom6_files.cmake")
 add_library(mom6_obj OBJECT ${mom6_src_files})
 set_target_properties(mom6_obj PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
-target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
-                                            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+
+if(REGIONAL_MOM6)
+  target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_symmetric>
+                                              $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+else()
+  target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
+                                              $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+endif()
 target_link_libraries(mom6_obj PRIVATE fms
                                        esmf
                                        stochastic_physics
@@ -42,8 +48,13 @@ target_link_libraries(mom6_obj PRIVATE fms
 add_library(mom6_nuopc_obj OBJECT ${mom6_nuopc_src_files})
 set_target_properties(mom6_nuopc_obj PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
-target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
-                                                  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+if(REGIONAL_MOM6)
+  target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_symmetric>
+                                                    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+else()
+  target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
+                                                    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+endif()
 target_link_libraries(mom6_nuopc_obj PRIVATE mom6_obj
                                              fms
                                              stochastic_physics

cmake/configure_apps.cmake

@@ -68,17 +68,19 @@ if(APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL)$")
   message("${APP_MSG} mode")
 endif()
 
-if(APP MATCHES "^(HAFS|HAFSW|HAFS-ALL)$")
+if(APP MATCHES "^(HAFS|HAFSW|HAFS-MOM6|HAFS-MOM6W|HAFS-ALL)$")
   set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
-  if(APP MATCHES "^(HAFS-ALL)$")
-    set(CDEPS    ON  CACHE BOOL "Enable CDEPS"               FORCE)
-    message("Configuring UFS app in HAFS with CDEPS mode")
-  endif()
+  set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
   set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
-  set(HYCOM      ON  CACHE BOOL "Enable HYCOM"               FORCE)
-  if(APP MATCHES "^(HAFSW|HAFS-ALL)$")
+  if(APP MATCHES "^(HAFS-MOM6|HAFS-MOM6W)$")
+    set(MOM6       ON  CACHE BOOL "Enable MOM6"              FORCE)
+  endif()
+  if(APP MATCHES "^(HAFS|HAFSW|HAFS-ALL)$")
+    set(HYCOM      ON  CACHE BOOL "Enable HYCOM"             FORCE)
+  endif()
+  if(APP MATCHES "^(HAFSW|HAFS-MOM6W|HAFS-ALL)$")
     set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
     message("Configuring UFS app in HAFS with Waves mode")
   endif()

tests/default_vars.sh

@@ -324,6 +324,7 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export POSTAPP='global'
 export USE_MERRA2=.false.
 
@@ -775,6 +776,7 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export FV3BMIC='p8c'
 export BMIC=.false.
 export DAYS=1
@@ -972,6 +974,7 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=true
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export DAYS=1
 
 # model configure
@@ -1047,6 +1050,7 @@ export HAFS=true
 export AQM=false
 export DATM_CDEPS=true
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export NTILES=1
@@ -1063,6 +1067,7 @@ export S2S=false
 export HAFS=true
 export AQM=false
 export DOCN_CDEPS=true
+export CDEPS_INLINE=false
 export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export NTILES=1
@@ -1081,6 +1086,7 @@ export HAFS=true
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export NTILES=1
@@ -1152,6 +1158,7 @@ export HAFS=true
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export NTILES=1

tests/fv3_conf/hafs_fv3_run.IN

@@ -17,6 +17,8 @@ elif [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm" ] || \
      [ ${CNTL_DIR} = "hafs_regional_specified_moving_1nest_atm" ] || \
      [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn" ] || \
      [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_debug" ] || \
+     [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
+     [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ] || \
      [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
   cp -r @[INPUTDATA_ROOT]/FV3_hafs_input_data/INPUT_hafs_regional_storm_following_1nest_atm/* ./INPUT/
 elif [ ${CNTL_DIR} = "hafs_regional_1nest_atm" ] || \
@@ -49,3 +51,16 @@ if [[ $POSTAPP = 'hafs' ]]; then
     cp ${PATHRT}/parm/postxconfig-NT-hafs.txt postxconfig-NT_FH00.txt
     cp ${PATHRT}/parm/params_grib2_tbl_new params_grib2_tbl_new
 fi
+
+if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ]; then
+  rm -rf OUTPUT
+  mkdir OUTPUT
+  cp -r @[INPUTDATA_ROOT]/FV3_hafs_input_data/MOM6_regional_input_data/* ./INPUT/
+  cp ${PATHRT}/parm/MOM_input_hafs ./MOM_input
+  cp ${PATHRT}/parm/MOM6_data_table_hafs ./data_table
+fi
+if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
+   [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ]; then
+  cp @[INPUTDATA_ROOT]/FV3_hafs_input_data/CDEPS_input_data/gfs_mesh.nc ./INPUT/
+  cp @[INPUTDATA_ROOT]/FV3_hafs_input_data/CDEPS_input_data/gfs_forcings.nc ./INPUT/
+fi

tests/fv3_conf/hafs_ww3_run.IN

@@ -1,10 +1,14 @@
 
 if [ ${CNTL_DIR} = "hafs_regional_atm_wav" ] || \
    [ ${CNTL_DIR} = "hafs_regional_atm_ocn_wav" ] || \
+   [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
+   [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ] || \
    [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
   cp @[INPUTDATA_ROOT_WW3]/@[WW3_MODDEF] ./mod_def.ww3
   cp @[INPUTDATA_ROOT_WW3]/@[MESH_WAV] ./mesh.hafs.nc
-  if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
+  if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
+     [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ] || \
+     [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
     cp @[INPUTDATA_ROOT]/FV3_hafs_input_data/WW3_hafs_regional_input_data/wind.natl_6m_2020082512 ./wind.ww3
   else
     cp @[INPUTDATA_ROOT_WW3]/wind.natl_6m ./wind.ww3

tests/fv3_conf/hycom_hat10_run.IN

@@ -1,6 +1,7 @@
 
 if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn" ] || \
    [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_debug" ] || \
+   [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
    [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
   rsync -arv @[INPUTDATA_ROOT]/FV3_hafs_input_data/HYCOM_hafs_regional_input_data/. ./
 else

tests/logs/OpnReqTests_control_p8_hera.log

@@ -1,9 +1,9 @@
-Wed Jan 31 19:20:35 UTC 2024
+Fri Feb  2 02:17:07 UTC 2024
 Start Operation Requirement Test
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_bit_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_24457/control_p8_gnu_bit_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_7057/control_p8_gnu_bit_base
 Checking test bit_base control_p8_gnu results ....
 Moving baseline bit_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -51,14 +51,14 @@ Moving baseline bit_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 283.367920
-  0: The maximum resident set size (KB)                   = 1304280
+  0: The total amount of wall time                        = 280.516405
+  0: The maximum resident set size (KB)                   = 1307816
 
 Test bit_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_dbg_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_24457/control_p8_gnu_dbg_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_7057/control_p8_gnu_dbg_base
 Checking test dbg_base control_p8_gnu results ....
 Moving baseline dbg_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -106,14 +106,14 @@ Moving baseline dbg_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 925.385783
-  0: The maximum resident set size (KB)                   = 1289116
+  0: The total amount of wall time                        = 915.713491
+  0: The maximum resident set size (KB)                   = 1292056
 
 Test dbg_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_24457/control_p8_gnu_dcp
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_7057/control_p8_gnu_dcp
 Checking test dcp control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -160,14 +160,14 @@ Checking test dcp control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 251.670387
-  0: The maximum resident set size (KB)                   = 1281048
+  0: The total amount of wall time                        = 251.429761
+  0: The maximum resident set size (KB)                   = 1278592
 
 Test dcp control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_24457/control_p8_gnu_mpi
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_7057/control_p8_gnu_mpi
 Checking test mpi control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -214,14 +214,14 @@ Checking test mpi control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 252.393805
-  0: The maximum resident set size (KB)                   = 1280764
+  0: The total amount of wall time                        = 250.829038
+  0: The maximum resident set size (KB)                   = 1281948
 
 Test mpi control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_24457/control_p8_gnu_rst
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_7057/control_p8_gnu_rst
 Checking test rst control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -268,14 +268,14 @@ Checking test rst control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 253.888089
-  0: The maximum resident set size (KB)                   = 1285060
+  0: The total amount of wall time                        = 250.474906
+  0: The maximum resident set size (KB)                   = 1285948
 
 Test rst control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_24457/control_p8_gnu_std_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_7057/control_p8_gnu_std_base
 Checking test std_base control_p8_gnu results ....
 Moving baseline std_base control_p8_gnu files ....
  Moving sfcf000.nc .........OK
@@ -323,14 +323,14 @@ Moving baseline std_base control_p8_gnu files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 249.734159
-  0: The maximum resident set size (KB)                   = 1283704
+  0: The total amount of wall time                        = 258.712522
+  0: The maximum resident set size (KB)                   = 1281208
 
 Test std_base control_p8_gnu PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_24457/control_p8_gnu_thr
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_7057/control_p8_gnu_thr
 Checking test thr control_p8_gnu results ....
  Comparing sfcf000.nc .........OK
  Comparing sfcf021.nc .........OK
@@ -377,11 +377,11 @@ Checking test thr control_p8_gnu results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 247.695087
-  0: The maximum resident set size (KB)                   = 1280276
+  0: The total amount of wall time                        = 248.574949
+  0: The maximum resident set size (KB)                   = 1279744
 
 Test thr control_p8_gnu PASS
 
 OPERATION REQUIREMENT TEST WAS SUCCESSFUL
-Wed Jan 31 22:36:38 UTC 2024
-Elapsed time: 03h:16m:04s. Have a nice day!
+Fri Feb  2 12:53:19 UTC 2024
+Elapsed time: 10h:36m:13s. Have a nice day!