Merge branch 'ufs-community:develop' into RT_bmark_gefs

doc/UsersGuide/Makefile

@@ -15,6 +15,11 @@ help:
 
 .PHONY: help Makefile linkcheck
 
+doc:
+	make clean
+	$(MAKE) linkcheck
+	$(MAKE) html
+
 linkcheck:
 	$(SPHINXBUILD) -b linkcheck $(SPHINXOPTS) $(SOURCEDIR) $(LINKCHECKDIR)
 

doc/UsersGuide/source/InputsOutputs.rst

@@ -793,7 +793,7 @@ The input files containing grid information and the time-varying forcing files f
 
 .. note:: 
 
-   Users can find atmospheric forcing files for use with the land (:ref:`LND <lnd>`) component in the `Land Data Assimilation (DA) data bucket <https://registry.opendata.aws/noaa-ufs-land-da/>`__. These files provide atmospheric forcing data related to precipitation, solar radiation, longwave radiation, temperature, pressure, winds, humidity, topography, and mesh data. Forcing files for the land component configuration come from the Global Soil Wetness Project Phase 3 (`GSWP3 <https://hydro.iis.u-tokyo.ac.jp/GSWP3/>`__) dataset. 
+   Users can find atmospheric forcing files for use with the land (:ref:`LND <lnd>`) component in the `Land Data Assimilation (DA) data bucket <https://registry.opendata.aws/noaa-ufs-land-da/>`_. These files provide atmospheric forcing data related to precipitation, solar radiation, longwave radiation, temperature, pressure, winds, humidity, topography, and mesh data. Forcing files for the land component configuration come from the Global Soil Wetness Project Phase 3 (`GSWP3 <https://hydro.iis.u-tokyo.ac.jp/GSWP3/>`_) dataset. 
 
    .. code-block:: console
 
@@ -1021,18 +1021,18 @@ AQM inputs defined in ``aqm.rc`` are listed and described in :numref:`Table %s <
 
 .. _lnd-in:
 
--------
-LND
--------
+--------------
+NOAH-MP (LND)
+--------------
 
-LND component datasets are available from the `Land Data Assimilation (DA) System data bucket <https://registry.opendata.aws/noaa-ufs-land-da/>`__ and can be retrieved using a ``wget`` command: 
+LND component datasets are available from the `Land Data Assimilation (DA) System data bucket <https://registry.opendata.aws/noaa-ufs-land-da/>`_ and can be retrieved using a ``wget`` command: 
 
 .. code-block:: console
 
-   wget https://noaa-ufs-land-da-pds.s3.amazonaws.com/current_land_da_release_data/v1.2.0/Landdav1.2.0_input_data.tar.gz
-   tar xvfz Landdav1.2.0_input_data.tar.gz
+   wget https://noaa-ufs-land-da-pds.s3.amazonaws.com/develop-20240501/Landda_develop_data.tar.gz
+   tar xvfz Landda_develop_data.tar.gz
 
-These files will be untarred into an ``inputs`` directory if the user does not specify a different name. They include data for Dec. 21, 2019. :numref:`Table %s <LndInputFiles>` describes the file types. In each file name, ``YYYY`` refers to a valid 4-digit year, ``MM`` refers to a valid 2-digit month, and ``DD`` refers to a valid 2-digit day of the month. 
+These files will be untarred into an ``inputs`` directory. They include data for Jan. 1-2, 2000. :numref:`Table %s <LndInputFiles>` describes the file types. In each file name, ``YYYY`` refers to a valid 4-digit year, ``MM`` refers to a valid 2-digit month, and ``DD`` refers to a valid 2-digit day of the month. 
 
 .. _LndInputFiles:
 
@@ -1062,19 +1062,19 @@ These files will be untarred into an ``inputs`` directory if the user does not s
 
        oro_C96.mx100.tile*.nc
      - Tiled static files that contain information on maximum snow albedo, slope type, soil color and type, substrate temperature, vegetation greenness and type, and orography (grid and land mask information). ``*`` stands for the grid tile number [1-6]. 
-     - Static/fixed files
-   * - grid_spec.nc
+     - FV3 fix files/Grid information
+   * - grid_spec.nc (aka C96.mosaic.nc)
      - Contains information on the mosaic grid
-     - Grid
+     - FV3 fix files/Grid information
    * - C96_grid.tile*.nc
-     - C96 grid information for tiles 1-6, where ``*`` is the grid tile number [1-6]. 
-     - Grid
+     - C96 grid information for tiles 1-6 at C96 grid resolution, where ``*`` is the grid tile number [1-6]. 
+     - FV3 fix files/Grid information
    * - C96_oro_data.tile*.nc / oro_C96.mx100.tileN.nc
      - Orography files that contain grid and land mask information, where ``*`` is the grid tile number [1-6]. ``mx100`` refers to the ocean resolution (100=1º).
-     - Grid
-   * - See :ref:`CDEPS <cdeps-in>` for information on atmospheric forcing files. 
+     - FV3 fix files/Grid information
+   * - See :ref:`CDEPS <cdeps-in>` for information on GSWP3 atmospheric forcing files. 
      - Atmospheric forcing
-     - CDEPS
+     - CDEPS/DATM
    * - ghcn_snwd_ioda_YYYYMMDD.nc
      - GHCN snow depth data assimilation files
      - DA
@@ -1088,9 +1088,9 @@ These files will be untarred into an ``inputs`` directory if the user does not s
 Static Datasets (i.e., *fix files*)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The static files (listed in :numref:`Table %s <LndInputFiles>`) include specific information on location, time, soil layers, and fixed (invariant) experiment parameters that are required for the land component to run. The data must be provided in :term:`netCDF` format.
+The fix files (listed in :numref:`Table %s <LndInputFiles>`) include specific information on location, time, soil layers, and fixed (invariant) experiment parameters that are required for the land component to run. The data must be provided in :term:`netCDF` format.
 
-The following static files are available in the ``inputs/UFS_WM/FV3_fix_tiled/C96/`` data directory (downloaded :ref:`above <lnd-in>`):
+The following fix files are available in the ``inputs/UFS_WM/FV3_fix_tiled/C96/`` data directory (downloaded :ref:`above <lnd-in>`):
 
 .. code-block:: 
 
@@ -1104,7 +1104,7 @@ The following static files are available in the ``inputs/UFS_WM/FV3_fix_tiled/C9
    oro_C96.mx100.tile*.nc
 
 where ``*`` refers to the tile number (1-6). 
-Details on the configuration variables included in this file are available in the :ref:`Land DA documentation <landda:InputFiles>`. 
+Details on the configuration variables included in these files are available in the :ref:`Land DA documentation <landda:InputFiles>`. 
 
 .. _lnd-grid-ic-files:
 
@@ -1148,6 +1148,7 @@ The configuration files used by the UFS Weather Model are listed here and descri
    * ``field_table``
    * ``model_configure``
    * ``ufs.configure``
+   * ``fd_ufs.yaml``
    * ``suite_[suite_name].xml`` (used only at build time)
    * ``datm.streams`` (used by CDEPS)
    * ``datm_in`` (used by CDEPS)
@@ -1675,6 +1676,34 @@ However, ``ufs.configure`` files for other configurations of the Weather Model a
 
 .. note:: The ``aoflux_code`` option is used to define the algorithm that will be used to calculate atmosphere-ocean fluxes. The possible options are ``cesm`` and ``ccpp``. If ``ccpp`` is selected then the suite file provided in the ``aoflux_ccpp_suite`` option is used to calculate atmosphere-ocean fluxes through the use of CCPP host model.
 
+
+.. _fd-ufs:
+
+-----------------
+``fd_ufs.yaml``
+-----------------
+
+The ``fd_ufs.yaml`` file contains a field dictionary to configure several fields that are used in import/export operations by different Earth modeling components in ESMF's NUOPC coupling system. It allows the sharing of coupling fields between components. Entries in the field dictionary are organized as YAML lists of maps. The NUOPC Field Dictionary data structure in the model code is set up by the NUOPC function called ``NUOPC_FieldDictionarySetup()``, which loads the ``fd_ufs.yaml`` file (see `UFSDriver.F90 <https://github.com/ufs-community/ufs-weather-model/blob/develop/driver/UFSDriver.F90>`_). The field
+metadata described in each entry are used by the NUOPC layer to match fields provided and requested by the various component models. The field dictionary can be shared with other Earth modeling systems that use the same ESPS coupling strategy, such as the Community Earth System Model (CESM).
+
+The standard field metadata for each coupling field has the following keys and corresponding values:
+
+.. code-block:: console
+
+   - standard_name: <field_name>
+     canonical_units: <unit>
+     description: <brief description about this field>
+     alias: <other_field_name>
+
+* ``standard_name`` (required): Name of the field. 
+* ``canonical_units`` (required): The units used to fully define the field
+* ``description`` (optional): Brief explanation of the field
+* ``alias`` (optional): Alternative names for the field. An alias can be one character string or a list of strings (e.g., ``<name>`` or ``[<name>, <name2>]``). This allows a field to have different names used in the coupling field exchange.
+
+Either the ``standard_name`` or ``alias`` name can be used in a component model, and the NUOPC layer will recognize these fields as the same field using the definitions provided in the YAML file. For more on the NUOPC field dictionary, visit the `documentation <https://earthsystemmodeling.org/docs/release/latest/NUOPC_refdoc/node3.html#SECTION00032000000000000000>`_.
+
+
+
 .. _SDF-file:
 
 ---------------------------------------

doc/UsersGuide/source/Introduction.rst

@@ -8,7 +8,7 @@ The Unified Forecast System (:term:`UFS`) Weather Model (:term:`WM`) is a progno
 used for short- and medium-range research and operational forecasts, as exemplified by
 its use in the operational Global Forecast System (GFS) of the National Oceanic and
 Atmospheric Administration (NOAA). In addition to its use in NOAA's operational forecast systems, the UFS WM is the atmospheric model used in public UFS application releases, such as the Short-Range Weather (SRW) Application v2.2.0 release. These releases represent a snapshot of a continuously evolving system undergoing open
-development. More information about the UFS can be found on the UFS Community Portal at https://ufscommunity.org/ and on the Earth Prediction Innovation Center (EPIC) website at https://epic.noaa.gov/get-code/ufs-weather-model/.
+development. More information about the UFS can be found on the UFS Community Portal at https://ufs.epic.noaa.gov/ and on the Earth Prediction Innovation Center (EPIC) website at https://epic.noaa.gov/get-code/ufs-weather-model/.
 
 Key architectural elements of the UFS WM, along with links to external detailed documentation
 for those elements, are listed below:

doc/UsersGuide/source/conf.py

@@ -14,13 +14,15 @@
 #
 import os
 import sys
+import sphinx
+from sphinx.util import logging
 sys.path.insert(0, os.path.abspath('.'))
-
+sys.path.insert(0, os.path.abspath('../../../tests-dev'))
 
 # -- Project information -----------------------------------------------------
 
 project = 'UFS Weather Model Users Guide'
-copyright = '2020'
+copyright = '2024'
 author = ' '
 
 # The short X.Y version
@@ -121,6 +123,7 @@
 def setup(app):
     app.add_css_file('custom.css')  # may also be an URL
     app.add_css_file('theme_overrides.css')  # may also be an URL
+    app.connect('autodoc-process-docstring', warn_undocumented_members) # Created warnings for undocumented portions of Python scripts
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -212,6 +215,34 @@ def setup(app):
 
 # -- Extension configuration -------------------------------------------------
 
+# -- Options for autodoc extension ---------------------------------------
+
+autodoc_mock_imports = [
+   ]
+
+logger = logging.getLogger(__name__)
+
+# Ensure that warnings pop up when functions, attributes, or methods are undocumented
+members_to_watch = ['function', 'attribute', 'method']
+def warn_undocumented_members(app, what, name, obj, options, lines):
+    if(what in members_to_watch and len(lines)==0):
+        message = what + " is undocumented: " + name + "(%d)"% len(lines)
+        logger.warning(message)
+
+autodoc_default_options = {
+    "members": True,
+    "undoc-members": True,
+    "show-inheritance": True,
+}
+
+add_module_names = False
+
+# -- Options for napoleon extension ---------------------------------------
+
+napoleon_numpy_docstring = False
+napoleon_google_docstring = True
+napoleon_custom_sections = [('Returns', 'params_style')] # Allows return of multiple values
+
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.

doc/UsersGuide/source/create_log.rst

@@ -0,0 +1,7 @@
+create\_log module
+==================
+
+.. automodule:: create_log
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/UsersGuide/source/create_xml.rst

@@ -0,0 +1,7 @@
+create\_xml module
+==================
+
+.. automodule:: create_xml
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/UsersGuide/source/index.rst

@@ -15,6 +15,7 @@ Welcome to the UFS Weather Model User's Guide
    BuildingAndRunning
    InputsOutputs
    Configurations
+   modules
    ConfigParameters
    AutomatedTesting
    FAQ

doc/UsersGuide/source/modules.rst

@@ -0,0 +1,9 @@
+Technical Documentation for ``tests-dev``
+==========================================
+
+.. toctree::
+   :maxdepth: 4
+
+   create_log
+   create_xml
+   ufs_test_utils

doc/UsersGuide/source/ufs_test_utils.rst

@@ -0,0 +1,7 @@
+ufs\_test\_utils module
+=======================
+
+.. automodule:: ufs_test_utils
+   :members:
+   :undoc-members:
+   :show-inheritance:

tests-dev/create_log.py

@@ -6,7 +6,7 @@
 from ufs_test_utils import get_testcase, write_logfile, delete_files, machine_check_off
 
 def finish_log():
-    """Collect regression test results and generate log file.
+    """Collects regression test results and generates log file.
     """
     UFS_TEST_YAML = str(os.getenv('UFS_TEST_YAML'))
     PATHRT     = os.getenv('PATHRT')

tests-dev/create_xml.py

@@ -5,18 +5,19 @@
 from ufs_test_utils import get_testcase, write_logfile, rrmdir, machine_check_off
 
 def rocoto_create_entries(RTPWD,MACHINE_ID,INPUTDATA_ROOT,INPUTDATA_ROOT_WW3,INPUTDATA_ROOT_BMIC,RUNDIR_ROOT,NEW_BASELINE,ROCOTO_XML):
-    """Generate header information for Rocoto xml file
+    """Generate header information for Rocoto XML file
 
     Args:
         RTPWD (str): Baseline directory
-        MACHINE_ID (str): Machine ID i.e. Hera, Gaea, Jet, etc.
+        MACHINE_ID (str): Machine ID i.e., Hera, Gaea, Jet, etc.
         INPUTDATA_ROOT (str): Input data directory
         INPUTDATA_ROOT_WW3 (str): WW3 input data directory
         INPUTDATA_ROOT_BMIC (str): BMIC input data directory
         RUNDIR_ROOT (str): Test run directory
         NEW_BASELINE (str): Directory for newly generated baselines
-        ROCOTO_XML (str): Rocoto .xml filename to write to
+        ROCOTO_XML (str): Rocoto XML filename to write to
     """
+
     PATHRT = os.getenv('PATHRT')
     LOG_DIR= PATHRT+'/logs/log_'+MACHINE_ID
     PATHTR, tail = os.path.split(PATHRT)
@@ -43,17 +44,17 @@ def rocoto_create_entries(RTPWD,MACHINE_ID,INPUTDATA_ROOT,INPUTDATA_ROOT_WW3,INP
     f.close()
 
 def rocoto_create_compile_task(MACHINE_ID,COMPILE_ID,ROCOTO_COMPILE_MAXTRIES,MAKE_OPT,ACCNR,COMPILE_QUEUE,PARTITION,ROCOTO_XML):
-    """Generate and append compile task into Rocoto xml file
+    """Generate and append compile task into Rocoto XML file
 
     Args:
-        MACHINE_ID (str): Machine ID i.e. Hera, Gaea, Jet, etc.
-        COMPILE_ID (str): Compile identifier e.g. s2swa_intel
+        MACHINE_ID (str): Machine ID i.e., Hera, Gaea, Jet, etc.
+        COMPILE_ID (str): Compile identifier e.g., s2swa_intel
         ROCOTO_COMPILE_MAXTRIES (str): Max attempts for compile
         MAKE_OPT (str): Make build options
         ACCNR (str): Account to run the job with
-        COMPILE_QUEUE (str): QOS i.e. batch, windfall, normal, etc.
-        PARTITION (str): System partition i.e. xjet, c5
-        ROCOTO_XML (str): Rocoto .xml filename to write to
+        COMPILE_QUEUE (str): Quality of Service (QOS), i.e., batch, windfall, normal, etc.
+        PARTITION (str): System partition i.e., xjet, c5
+        ROCOTO_XML (str): Rocoto XML filename to write to
     """
     NATIVE=""
     BUILD_CORES="8"
@@ -90,11 +91,11 @@ def rocoto_create_compile_task(MACHINE_ID,COMPILE_ID,ROCOTO_COMPILE_MAXTRIES,MAK
     f.close()
 
 def write_metatask_begin(COMPILE_METATASK_NAME, filename):
-    """Write compile task metadata to Rocoto xml file
+    """Write compile task metadata to Rocoto XML file
 
     Args:
-        COMPILE_METATASK_NAME (str): Compile job name e.g. s2swa_intel
-        filename (str): Rocoto xml filename to append to
+        COMPILE_METATASK_NAME (str): Compile job name e.g., s2swa_intel
+        filename (str): Rocoto XML filename to append to
     """
     metatask_name = f"""  <metatask name="compile_{COMPILE_METATASK_NAME}_tasks"><var name="zero">0</var>
 """
@@ -103,10 +104,10 @@ def write_metatask_begin(COMPILE_METATASK_NAME, filename):
     f.close()
 
 def write_metatask_end(filename):
-    """Append closing metatask element to Rocoto xml
+    """Append closing metatask element to Rocoto XML
 
     Args:
-        filename (str): Rocoto xml filename
+        filename (str): Rocoto XML filename
     """
     metatask_name = f"""  </metatask>
 """
@@ -118,10 +119,10 @@ def write_compile_env(SCHEDULER,PARTITION,JOB_NR,COMPILE_QUEUE,RUNDIR_ROOT):
     """Generate compile task .env file
 
     Args:
-        SCHEDULER (str): Job scheduler e.g. pbs, slurm
-        PARTITION (str): System partition i.e. xjet, c5
+        SCHEDULER (str): Job scheduler, e.g., pbs, slurm
+        PARTITION (str): System partition, i.e., xjet, c5
         JOB_NR (str): Job number
-        COMPILE_QUEUE (str): QOS i.e. batch, windfall, normal, etc.
+        COMPILE_QUEUE (str): Quality of Service (QOS), i.e., batch, windfall, normal, etc.
         RUNDIR_ROOT (str): Test run directory
     """
     filename   = RUNDIR_ROOT+"/compile_"+str(os.getenv('COMPILE_ID'))+".env"
@@ -308,6 +309,11 @@ def make_loghead(ACCNR,MACHINE_ID,RUNDIR_ROOT,RTPWD,REGRESSIONTEST_LOG):
         write_logfile(filename, "a", output="* (-v) - VERBOSE OUTPUT"+"\n")
 
 def xml_loop():
+    """
+    Creates workflow XML file for each test.
+    Reads in ``baseline_setup.yaml``, parses ``ufs_test.yaml``, and sets up Rocoto XML 
+    according to the runtime arguments given in ``ufs_test.sh``. 
+    """
     ACCNR      = str(os.getenv('ACCNR'))
     PATHRT     = str(os.getenv('PATHRT'))
     MACHINE_ID = str(os.getenv('MACHINE_ID'))