Merge remote-tracking branch 'origin/develop' into cubed_sphere_chunks

.gitmodules

@@ -48,6 +48,10 @@
   path = NOAHMP-interface/noahmp
   url = https://github.com/NOAA-EMC/noahmp
   branch = develop
+[submodule "LM4-driver"]
+  path = LM4-driver
+  url = https://github.com/NOAA-GFDL/LM4-NUOPC-driver
+  branch = develop
 [submodule "fire_behavior"]
   path = fire_behavior
   url = https://github.com/NOAA-EMC/fire_behavior

CMakeLists.txt

@@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML ATMF LND S2S S2SA S2SW S2SWA S2SWAL ATM_DS2S ATM_DS2S-PCICE HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
+list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML ATMF LND LND-LM4 S2S S2SA S2SW S2SWA S2SWAL ATM_DS2S ATM_DS2S-PCICE HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -29,6 +29,7 @@ set(UFS_GOCART      OFF CACHE BOOL "Enable GOCART")
 set(MOM6            OFF CACHE BOOL "Enable MOM6")
 set(HYCOM           OFF CACHE BOOL "Enable HYCOM")
 set(CICE6           OFF CACHE BOOL "Enable CICE6")
+set(LM4             OFF CACHE BOOL "Enable LM4")
 set(WW3             OFF CACHE BOOL "Enable WW3")
 set(STOCH_PHYS      OFF CACHE BOOL "Enable Stochastic Physics")
 set(CMEPS           OFF CACHE BOOL "Enable CMEPS")
@@ -48,6 +49,7 @@ message("GOCART ........... ${UFS_GOCART}")
 message("MOM6 ............. ${MOM6}")
 message("HYCOM ............ ${HYCOM}")
 message("CICE6 ............ ${CICE6}")
+message("LM4 .............. ${LM4}")
 message("WW3 .............. ${WW3}")
 message("STOCH_PHYS ....... ${STOCH_PHYS}")
 message("CDEPS ............ ${CDEPS}")
@@ -157,7 +159,7 @@ if(FMS)
   elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|ATM_DS2S|ATM_DS2S-PCICE|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$")
     add_library(fms ALIAS FMS::fms_r8)
   endif()
-  if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|ATMF|HAFS|HAFS-ALL)$")
+  if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|ATMF|ATML-LM4|LND-LM4|HAFS|HAFS-ALL)$")
     if(32BIT)
       add_library(fms ALIAS FMS::fms_r4)
     else()
@@ -256,8 +258,13 @@ if(CDEPS)
 endif()
 
 ###############################################################################
-### Land Components [NOAHMP]
+### Land Components [NOAHMP and LM4]
 ###############################################################################
+
+if (LM4)
+  add_subdirectory(LM4-driver)
+endif()
+
 if(NOAHMP)
   add_subdirectory(NOAHMP-interface)
 endif()
@@ -329,6 +336,12 @@ if(CICE6)
   list(APPEND _ufs_libs_public cice)
 endif()
 
+if (LM4)
+   add_dependencies(ufs lm4)
+   list(APPEND _ufs_defs_private FRONT_LM4=lm4_cap_mod)
+   list(APPEND _ufs_libs_public lm4)
+endif()
+
 if(CMEPS)
   add_dependencies(ufs cmeps)
   list(APPEND _ufs_defs_private CMEPS FRONT_CMEPS=MED)

README.md

@@ -25,6 +25,7 @@ The top level directory structure groups source code and input files as follow:
 | ```FV3/```                | Contains FV3 atmosphere model component including FV3 dynamical core, dynamics to physics driver, physics and IO. |
 | ```GOCART/```             | Contains GOCART aerosol model component |
 | ```HYCOM-interface/```    | Contains HYCOM ocean model component |
+| ```LM4-driver/```         | Contains LM4 land component |
 | ```MOM6-interface/```     | Contains MOM6 ocean model component |
 | ```NOAHMP-interface/```   | Contains Noah-MP land model component |
 | ```WW3/```                | Contains community wave modeling framework WW3 |

cmake/configure_apps.cmake

@@ -13,7 +13,7 @@
 ###############################################################################
 ### Configure Application Components
 ###############################################################################
-if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML|ATMF)$")
+if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML|ATMF|ATML-LM4)$")
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
   set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
@@ -31,6 +31,10 @@ if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML|ATMF)$")
     set(CMEPS    ON  CACHE BOOL "Enable CMEPS"               FORCE)
     set(NOAHMP   ON  CACHE BOOL "Enable NOAHMP"              FORCE)
     message("Configuring UFS app in Atmosphere with Air Quality mode")
+  elseif(APP MATCHES "ATML-LM4")
+    set(CMEPS    ON  CACHE BOOL "Enable CMEPS"               FORCE)
+    set(LM4      ON  CACHE BOOL "Enable LM4"                 FORCE)	    
+    message("Configuring UFS app in Atmosphere with Air Quality mode")
   elseif(APP MATCHES "ATMF")
     set(FIRE_BEHAVIOR ON CACHE BOOL "Enable Fire Behavior"   FORCE)
   else()
@@ -116,3 +120,10 @@ if(APP MATCHES "^(LND)$")
   set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
   set(NOAHMP     ON  CACHE BOOL "Enable NOAHMP"              FORCE)
 endif()
+
+if(APP MATCHES "^(LND-LM4)$")
+  set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
+  set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
+  set(LM4        ON  CACHE BOOL "Enable LM4"                 FORCE)
+  set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
+endif()

doc/UsersGuide/source/CodeOverview.rst

@@ -86,6 +86,8 @@ The UFS :term:`WM` repository supports the :term:`UFS` short- and medium-range w
     - https://github.com/NOAA-EMC/AQM
   * - Noah-MP Land Surface Model (Noah-MP)
     - https://github.com/NOAA-EMC/noahmp
+  * - NOAA-GFDL Land Model (:term:`LM4`)
+    -  
 
 In the table, the left-hand column contains a description of each repository, and the 
 right-hand column shows the GitHub location of the authoritative component repositories. 
@@ -143,6 +145,9 @@ The umbrella repository for the UFS WM is named ``ufs-weather-model``. Under thi
     │    └── HYCOM                               -------- HYCOM ocean model
     │        └── (NUOPC)                         -------- NUOPC HYCOM cap
     ├── LICENSE.md
+    ├── LM4-driver 
+    │   ├── (LM4)                                -------- NUOPC NOAA-GFDL Land Model version 4
+    │   └── (nuopc_cap)                          -------- NUOPC LM4 cap
     ├── modulefiles                              -------- system module files for supported HPC systems
     ├── MOM6-interface
     │    └── MOM6

doc/UsersGuide/source/Glossary.rst

@@ -97,6 +97,9 @@ Glossary
    HYCOM
       The HYbrid Coordinate Ocean Model (`HYCOM <https://www.hycom.org/>`__) was developed to address known shortcomings in the vertical coordinate scheme of the Miami Isopycnic-Coordinate Ocean Model (MICOM). HYCOM is a primitive equation, general circulation model with vertical coordinates that remain isopycnic in the open, stratified ocean. However, the isopycnal vertical coordinates smoothly transition to z-coordinates in the weakly stratified upper-ocean mixed layer, to terrain-following sigma coordinates in shallow water regions, and back to z-level coordinates in very shallow water. The latter transition prevents layers from becoming too thin where the water is very shallow. See the `HYCOM User's Guide <https://www.hycom.org/attachments/063_hycom_users_guide.pdf>`__ for more information on using the model. The `HYCOM model code <https://github.com/NOAA-EMC/HYCOM-src>`__ is publicly available on GitHub. 
 
+   LM4
+      NUOPC NOAA-GFDL Land Model version 4
+
    LND
    land component
       The Noah Multi-Physics (Noah-MP) land surface model (LSM) is an open-source, community-developed LSM that has been incorporated into the UFS Weather Model (WM). It is the UFS WM's land component. 

driver/UFSDriver.F90

@@ -79,6 +79,9 @@ MODULE UFSDriver
 #ifdef FRONT_NOAH
       use FRONT_NOAH,       only: NOAH_SS  => SetServices
 #endif
+#ifdef FRONT_LM4
+      use FRONT_LM4,        only: LM4_SS  => SetServices
+#endif      
 #ifdef FRONT_NOAHMP
       use FRONT_NOAHMP,     only: NOAHMP_SS  => SetServices
 #endif
@@ -467,6 +470,14 @@ subroutine SetModelServices(driver, rc)
             found_comp = .true.
           end if
 #endif
+#ifdef FRONT_LM4
+          if (trim(model) == "lm4") then
+            call NUOPC_DriverAddComp(driver, trim(prefix), LM4_SS, &
+              petList=petList, comp=comp, rc=rc)
+            if (ChkErr(rc,__LINE__,u_FILE_u)) return
+            found_comp = .true.
+          end if
+#endif 
 #ifdef FRONT_NOAHMP
           if (trim(model) == "noahmp") then
             call NUOPC_DriverAddComp(driver, trim(prefix), NOAHMP_SS, &

modulefiles/ufs_gaea.intel.lua

@@ -7,10 +7,10 @@ whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]=
 
 prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
 
-stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
 load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
 stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"

modulefiles/ufs_orion.intel.lua

@@ -15,6 +15,9 @@ load(pathJoin("cmake", cmake_ver))
 
 load("ufs_common")
 
+-- HDF5 needed for LM4
+hdf5_ver=os.getenv("hdf5_ver") or "1.14.0"
+load(pathJoin("hdf5", hdf5_ver))
 nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
 load(pathJoin("nccmp", nccmp_ver))
 

tests/bl_date.conf

@@ -1 +1 @@
-export BL_DATE=20241001
+export BL_DATE=20241011

tests/default_vars.sh

@@ -1295,6 +1295,7 @@ export INPUT_NML=global_control.nml.IN
 export FIELD_TABLE=field_table_thompson_noaero_tke_GOCART
 export DIAG_TABLE=diag_table_cpld.IN
 export DIAG_TABLE_ADDITIONAL=''
+export FIELD_TABLE_ADDITIONAL=''
 export FV3_RUN=cpld_control_run.IN
 export TILEDFIX=.false.
 

tests/fv3_conf/fv3_slurm.IN_gaea

@@ -28,6 +28,7 @@ export OMP_STACKSIZE=1024M
 export NC_BLKSZ=1M
 export ESMF_RUNTIME_PROFILE=ON
 export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
+export FI_VERBS_PREFER_XRC=0
 
 # Avoid job errors because of filesystem synchronization delays
 sync && sleep 1

tests/fv3_conf/lm4_run.IN

@@ -0,0 +1,97 @@
+mkdir -p INPUT RESTART 
+
+
+if [[ $atm_model == 'datm' ]]; then
+# need to make some adjustments for LM4's grid file requirements
+# assume LNDRES is defined
+
+  if [[ $LNDRES == 'C96' ]]; then
+    FV3_DIR=FV3_input_data
+  else
+    FV3_DIR=FV3_input_data${LNDRES#C}
+  fi
+
+  ## get these if don't already have:
+  rsync -arv @[INPUTDATA_ROOT]/CPL_FIX/a@[LNDRES]o@[OCNRES]/grid_spec.nc ./INPUT
+  rsync -arv @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT/@[LNDRES]_grid*.nc ./INPUT
+  rsync -arv @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT/grid_spec.nc ./INPUT/@[LNDRES]_mosaic.nc
+  rsync -arv @[INPUTDATA_ROOT]/MOM6_FIX/@[OCNRES]/* ./INPUT
+
+elif [[ $atm_model == 'fv3' ]]; then
+  # assume ATMRES is defined
+  if [[ $ATMRES == 'C96' ]]; then
+    FV3_DIR=FV3_input_data
+  else
+    FV3_DIR=FV3_input_data${ATMRES#C}
+  fi
+
+fi  
+
+
+
+## LM4
+
+# out of the box, LM4 expects aXl referenced in grid_spec.nc for grid land area. Include them for now.
+
+# check if ATMRES or LNDRES is equal to C96:
+if [[ $ATMRES == 'C96' || $LNDRES == 'C96' ]]; then
+  tar -xf @[INPUTDATA_LM4]/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar -C ./INPUT/ \
+    --wildcards "C96_mosaic_tile?XC96_mosaic_tile?.nc"
+
+  tar -xvf @[INPUTDATA_LM4]/c96_LM4/c96_topo_rough_land.nc.tar -C ./INPUT/
+else
+   echo 'Only setup for C96 is supported'
+fi
+
+ln -s @[INPUTDATA_LM4]/common_LM4/biodata.nc                 ./INPUT/
+ln -s @[INPUTDATA_LM4]/common_LM4/cover_type.nc              ./INPUT/
+ln -s @[INPUTDATA_LM4]/common_LM4/geohydrology.nc            ./INPUT/
+ln -s @[INPUTDATA_LM4]/common_LM4/geohydrology_table_2a2n.nc ./INPUT/
+ln -s @[INPUTDATA_LM4]/common_LM4/ground_type.nc             ./INPUT/
+ln -s @[INPUTDATA_LM4]/common_LM4/landuse.nc                 ./INPUT/
+ln -s @[INPUTDATA_LM4]/common_LM4/soil_brdf.nc               ./INPUT/
+ln -s @[INPUTDATA_LM4]/common_LM4/soil_type_hwsd_5minute.nc  ./INPUT/soil_type.nc
+
+
+cd INPUT
+rsync -arv @[INPUTDATA_ROOT]/DATM_GSWP3_input_data/. .
+cd -
+
+SUFFIX=${RT_SUFFIX}
+# restart
+if [ $WARM_START = .true. ]; then
+  # # NoahMP restart files
+  # cp ../${DEP_RUN}${SUFFIX}/ufs.cpld.lnd.out.${RESTART_FILE_SUFFIX_SECS}.tile*.nc RESTART/.
+
+  # LM4 restart files
+  # expected checkpoint restarts are named like: YYYYMMDD.HHMMSS.*.res.tile?.nc
+
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.landuse.res        ./INPUT/
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.cana.res.tile*.nc  ./INPUT/
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.glac.res.tile*.nc  ./INPUT/
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.lake.res.tile*.nc  ./INPUT/
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.land.res.tile*.nc  ./INPUT/
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.snow.res.tile*.nc  ./INPUT/
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.soil.res.tile*.nc  ./INPUT/
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.vegn1.res.tile*.nc ./INPUT/
+  cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${LM4_RESTART_PREFIX}.vegn2.res.tile*.nc ./INPUT/
+
+  # remove datestamp
+  rename ${LM4_RESTART_PREFIX}. "" INPUT/${LM4_RESTART_PREFIX}.*.res*
+
+  # CMEPS restart and pointer files
+  RFILE1=ufs.cpld.cpl.r.${RESTART_FILE_SUFFIX_SECS}.nc
+  cp ../${DEP_RUN}${SUFFIX}/RESTART/${RFILE1} RESTART/.  
+  ls -1 "RESTART/${RFILE1}">rpointer.cpl
+
+  # CDEPS restart and pointer files
+  RFILE2=ufs.cpld.datm.r.${RESTART_FILE_SUFFIX_SECS}.nc
+  cp ../${DEP_RUN}${SUFFIX}/${RFILE2} RESTART/.
+  ls -1 "RESTART/${RFILE2}">rpointer.atm
+fi
+
+
+
+
+
+