Debug issues with preprocessor parsing pod settings file

diagnostics/WWEs/WWEs.html

@@ -0,0 +1,21 @@
+<!-- This file is part of the taux_diag module of the MDTF code package (see LICENSE.txt) -->
+
+<title>Mean zonal wind stress</title>
+<img src="../mdtf_diag_banner.png">
+<h3>Mean zonal wind stress</h3>
+<p>
+The zonal wind stress diagnostic module computes the annual cycle of
+zonal wind stress near the equator using multiple years of daily zonal
+wind stress output from climate models. The result is plotted as a
+latitudinally averaged longitude vs. time figure (i.e., Hovmoller). 
+</p>
+<p>
+</p>
+<TABLE>
+<TR>
+<TH align=left><font color=navy>Mean zonal wind stress
+<TH align=left>{{CASENAME}}
+<TR>
+<TH align=left>Zonal wind stress (Pa)
+<TH align=center><A href=model/{{CASENAME}}.Mean_TAUU.png>plot</A>
+</TABLE>

diagnostics/WWEs/WWEs.py

@@ -0,0 +1,149 @@
+import matplotlib
+matplotlib.use("Agg")  # non-X windows backend
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import xarray as xr
+import glob
+import os
+import time 
+import xesmf as xe
+import scipy
+from scipy import stats
+from functools import partial
+import intake
+import sys
+import yaml
+
+from WWE_diag_tools import (
+    land_mask_using_etopo,
+    regridder_model2obs,
+    nharm,
+    calc_raw_and_smth_annual_cycle,
+    isolate_WWEs,
+    WWE_characteristics,
+    WWE_statistics, #We don't need to do the statistics to make the likelihood by longitude plot
+    find_WWE_time_lon,
+    plot_model_Hovmollers_by_year)
+
+print("\n=======================================")
+print("BEGIN WWEs.py ")
+print("=======================================")
+
+def _preprocess(x, lon_bnds, lat_bnds):
+    return x.sel(lon=slice(*lon_bnds), lat=slice(*lat_bnds))
+
+work_dir  = os.environ["WORK_DIR"]
+obs_dir   = os.environ["OBS_DATA"]
+casename  = os.environ["CASENAME"]
+first_year= os.environ["first_yr"]
+last_year = os.environ["last_yr"]
+
+###########################################################################
+##############Part 1: Get, Plot Observations ##############################
+###########################################################################
+print(f'*** Now working on obs data\n------------------------------')
+obs_file_WWEs = obs_dir + '/TropFlux_120-dayHPfiltered_tauu_1980-2014.nc'
+
+print(f'*** Reading obs data from {obs_file_WWEs}')
+obs_WWEs    = xr.open_dataset(obs_file_WWEs)
+print(obs_WWEs, 'obs_WWEs')
+
+# Subset the data for the user defined first and last years #
+obs_WWEs = obs_WWEs.sel(time=slice(first_year, last_year))
+
+obs_lons = obs_WWEs.lon
+obs_lats = obs_WWEs.lat
+obs_time = obs_WWEs.time
+Pac_lons = obs_WWEs.Pac_lon
+obs_WWE_mask        = obs_WWEs.WWE_mask
+TropFlux_filt_tauu  = obs_WWEs.filtered_tauu
+TropFlux_WWEsperlon = obs_WWEs.WWEs_per_lon
+
+plot_model_Hovmollers_by_year(data = TropFlux_filt_tauu, wwe_mask = obs_WWE_mask,
+                                  lon_vals = Pac_lons, tauu_time = obs_time,
+                                  savename = f"{work_dir}/obs/PS/TropFlux_",
+                                  first_year = first_year, last_year = last_year)
+
+###########################################################################
+###########Parse MDTF-set environment variables############################
+###########################################################################
+#These variables come from the case_env_file that the framework creates
+#the case_env_file points to the csv file, which in turn points to the data files.
+#Variables from the data files are then read in. See example_multicase.py
+print("*** Parse MDTF-set environment variables ...")
+
+case_env_file = os.environ["case_env_file"]
+assert os.path.isfile(case_env_file), f"case environment file not found"
+with open(case_env_file, 'r') as stream:
+    try:
+        case_info = yaml.safe_load(stream)
+    except yaml.YAMLError as exc:
+        print(exc)
+
+cat_def_file = case_info['CATALOG_FILE']
+case_list    = case_info['CASE_LIST']
+# all cases share variable names and dimension coords in this example, so just get first result for each
+tauu_var   = [case['tauu_var'] for case in case_list.values()][0]
+time_coord = [case['time_coord'] for case in case_list.values()][0]
+lat_coord  = [case['lat_coord'] for case in case_list.values()][0]
+lon_coord  = [case['lon_coord'] for case in case_list.values()][0]
+
+# open the csv file using information provided by the catalog definition file
+cat = intake.open_esm_datastore(cat_def_file)
+
+# filter catalog by desired variable and output frequency
+tauu_subset = cat.search(variable_id=tauu_var, frequency="day")
+
+# examine assets for a specific file
+#tas_subset['CMIP.synthetic.day.r1i1p1f1.day.gr.atmos.r1i1p1f1.1980-01-01-1984-12-31'].df
+
+#Use partial function to only load part of the data file
+lon_bnds, lat_bnds = (0, 360), (-32.5, 32.5)
+partial_func       = partial(_preprocess, lon_bnds=lon_bnds, lat_bnds=lat_bnds)
+
+# convert tas_subset catalog to an xarray dataset dict
+tauu_dict = tauu_subset.to_dataset_dict(preprocess = partial_func,
+    xarray_open_kwargs={"decode_times": True, "use_cftime": True}
+)
+
+tauu_arrays = {}
+for k, v in tauu_dict.items(): 
+    arr = tauu_dict[k][tauu_var]
+    arr = arr.sel(lon = slice(120,280), lat = slice(-2.5, 2.5),
+                      time = slice(first_year, last_year))
+    arr = arr.mean(dim = (tauu_dict[k][lat_coord].name,tauu_dict[k][time_coord].name))
+
+    tauu_arrays[k] = arr
+
+###########################################################################
+# Part 3: Make a plot that contains results from each case
+# --------------------------------------------------------
+
+# set up the figure
+fig = plt.figure(figsize=(12, 4))
+ax = plt.subplot(1, 1, 1)
+
+# loop over cases
+for k, v in tauu_arrays.items():
+    v.plot(ax=ax, label=k)
+
+# add legend
+plt.legend()
+
+# add title
+plt.title("Mean Zonal Wind Stress")
+
+assert os.path.isdir(f"{work_dir}/model/PS"), f'Assertion error: {work_dir}/model/PS not found'
+plt.savefig(f"{work_dir}/model/PS/{casename}.Mean_TAUU.eps", bbox_inches="tight")
+
+# Part 4: Close the catalog files and
+# release variable dict reference for garbage collection
+# ------------------------------------------------------
+cat.close()
+tauu_dict = None
+# Part 5: Confirm POD executed successfully
+# ----------------------------------------
+print("Last log message by example_multicase POD: finished successfully!")
+sys.exit(0)

diagnostics/WWEs/WWEs_run_config.yml

@@ -0,0 +1,69 @@
+# Runtime yaml configuration file template for the MDTF-diagnostics package
+# Create a copy of this file for personal use, and pass it to the framework
+# with the -f | --configfile flag
+
+# List of POD(s) to run
+pod_list:
+  - "WWEs"
+
+# Case list entries (must be unique IDs for each simulation)
+case_list:
+  "CESM2_historical_r1i1p1f1_gn" :
+    model: "CESM2"
+    convention: "CMIP"
+    startdate: "19800101"
+    enddate: "19891231"
+
+#  "tauu_Eday_CESM2_historical_r1i1p1f1_gn_19900101-19991231" :
+#    model: "CESM2"
+#    convention: "CMIP"
+#    startdate: "19900101000000"
+#    enddate: "19991231000000"
+
+### Data location settings ###
+# Required: full or relative path to ESM-intake catalog header file
+DATA_CATALOG: "./diagnostics/WWEs/esm_catalog_CESM2_tauu_historical_r1i1p1f1.json"
+# Optional: Parent directory containing observational data used by individual PODs.
+# If defined, the framework assumes observational data is in OBS_DATA_ROOT/[POD_NAME]
+OBS_DATA_ROOT: "../inputdata/obs_data"
+# Required: Working directory location. Temporary files are written here.
+# Final output is also written here if the OUTPUT_DIR is not defined.
+WORK_DIR: "../wkdir"
+# Optional: Location to write final output if you don't want it in the wkdir
+OUTPUT_DIR: "../wkdir"
+### Environment Settings ###
+# Required: Location of the Anaconda/miniconda installation to use for managing
+# dependencies (path returned by running `conda info --base`.)
+conda_root: "/Users/eriley/anaconda3"
+# Optional: Directory containing the framework-specific conda environments. This should
+# be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left
+# blank, the framework will look for its environments in conda_root/envs
+conda_env_root: "/Users/eriley/anaconda3/envs"
+# Location of micromamba executable; REQUIRED if using micromamba
+micromamba_exe: ""
+### Data type settings ###
+# set to true to handle data files > 4 GB
+large_file: False
+### Output Settings ###
+# Set to true to have PODs save postscript figures in addition to bitmaps.
+save_ps: True
+# If true, leave pp data in OUTPUT_DIR after preprocessing; if false, delete pp data after PODs
+# run to completion
+save_pp_data: True
+# Set to true to perform data translation; default is True:
+translate_data: True
+# Set to true to save HTML and bitmap plots in a .tar file.
+make_variab_tar: False
+# Set to true to overwrite results in OUTPUT_DIR; otherwise results saved
+# under a unique name.
+overwrite: False
+# Generate html output for multiple figures per case
+"make_multicase_figure_html": False
+### Developer settings ###
+# Set to True to run the preprocessor
+run_pp: False
+# Additional custom preprocessing scripts to run on data
+# place these scripts in the MDTF-diagnostics/user_scripts directory
+# The framework will run the specified scripts whether run_pp is set to True or False
+user_pp_scripts:
+  - ""

diagnostics/WWEs/doc/WWE.rst

@@ -0,0 +1,60 @@
+.. This is a comment in RestructuredText format (two periods and a space).
+Westerly Wind Event Diagnostic Documentation
+================================
+
+Last update: 08/27/2024
+
+The zonal surface stress diagnostic makes a longitude vs. time plot of
+the equator zonal surface stress. Observations from TropFlux are
+compared to climate model output.
+
+Version & Contact info
+----------------------
+
+- PI: Emily M. Riley Dellaripa ([email protected]), Colorado State University
+- Current Developer: Emily M. Riley Dellaripa ([email protected]), Colorado State University
+- Contributors: Charlotte A. DeMott (CSU); Eric D. Maloney (CSU)
+
+
+Open source copyright agreement
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The MDTF framework is distributed under the LGPLv3 license (see LICENSE.txt). 
+
+Functionality
+-------------
+
+The main script generates the longitude vs. time plot of zonal surface
+stress.
+
+Required programming language and libraries
+-------------------------------------------
+
+This POD is written in Python 3.10 and requires the os, numpy, xarray,
+and matplotlib Python packages. These dependencies are included in the
+python3_base environment provided by the automated installation script
+for the MDTF Framework.
+
+Required model output variables
+-------------------------------
+
+This POD requires the zonal wind stress (TAUX) in 3D (time-lat-lon) in
+units of Pa.
+
+References
+----------
+1.Riley Dellaripa et al. (2024): Evaluation of Equatorial Westerly
+Wind Events in the Pacific Ocean in CMIP6 Models. *J. Climate*,
+`doi: 10.1175/JCLI-D-23-0629.1 < https://doi.org/10.1175/JCLI-D-23-0629.1>`__.
+
+
+More about this diagnostic
+--------------------------
+
+
+Links to external sites
+^^^^^^^^^^^^^^^^^^^^^^^
+
+
+More references and citations
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

diagnostics/WWEs/esm_catalog_CESM2_tauu_historical_r1i1p1f1.csv

@@ -0,0 +1,3 @@
+activity_id,branch_method,branch_time_in_child,branch_time_in_parent,experiment,experiment_id,frequency,grid,grid_label,institution_id,nominal_resolution,parent_activity_id,parent_experiment_id,parent_source_id,parent_time_units,parent_variant_label,product,realm,source_id,source_type,sub_experiment,sub_experiment_id,table_id,variable_id,variant_label,member_id,standard_name,long_name,units,vertical_levels,init_year,start_time,end_time,time_range,path,version
+CMIP,standard,674885,219000,all-forcing simulation of the recent past,historical,day,native 0.9x1.25 finite volume grid (192x288 latxlon),gn,NCAR,100 km,CMIP,piControl,CESM2,days since 0001-01-01 00:00:00,r1i1p1f1,model-output,atmos,CESM2,AOGCM BGC,none,none,Eday,tauu,r1i1p1f1,r1i1p1f1,surface_downward_eastward_stress,Surface Downward Eastward Wind Stress,Pa,1,,1980-01-01,1989-12-31,1980-01-01 00:00:00-1989-12-31 00:00:00,/Users/eriley/NOAA_POD/model_data/CESM2/tauu/tauu_Eday_CESM2_historical_r1i1p1f1_gn_19800101-19891231.nc,v0
+CMIP,standard,674885,219000,all-forcing simulation of the recent past,historical,fx,native 0.9x1.25 finite volume grid (192x288 latxlon),gn,NCAR,100 km,CMIP,piControl,CESM2,days since 0001-01-01 00:00:00,r1i1p1f1,model-output,atmos,CESM2,AOGCM BGC,none,none,fx,sftlf,r1i1p1f1,r1i1p1f1,land_area_fraction,Percentage of the grid  cell occupied by land (including lakes),%,1,,,,,/Users/eriley/NOAA_POD/model_data/sftlf_historical/CESM2/sftlf_fx_CESM2_historical_r1i1p1f1_gn.nc,v0