Yaml tables for ensembles and nests

data_override/include/data_override.inc

@@ -45,7 +45,7 @@ use mpp_domains_mod, only : domainUG, mpp_pass_SG_to_UG, mpp_get_UG_SG_domain, N
 use time_manager_mod, only: time_type, OPERATOR(>), OPERATOR(<)
 use fms2_io_mod,     only : FmsNetcdfFile_t, open_file, close_file, &
                             read_data, fms2_io_init, variable_exists, &
-                            get_mosaic_tile_file, file_exists
+                            get_mosaic_tile_file, file_exists, get_instance_filename
 use get_grid_version_mod, only: get_grid_version_1, get_grid_version_2
 use fms_string_utils_mod, only: string
 
@@ -591,9 +591,18 @@ subroutine read_table_yaml(data_table)
     integer :: nentries, mentries
     integer :: i
     character(len=50) :: buffer
+    character(len=255) :: filename !< Name of the expected data_table.yaml
     integer :: file_id
 
-    file_id = open_and_parse_file("data_table.yaml")
+    ! If doing and ensemble or nest run add the filename appendix (ens_XX or nest_XX) to the filename
+    call get_instance_filename("data_table.yaml", filename)
+    if (index(trim(filename), "ens_") .ne. 0) then
+      if (file_exists(filename) .and. file_exists("data_table.yaml")) &
+        call mpp_error(FATAL, "Both data_table.yaml and "//trim(filename)//" exists, pick one!")
+    endif
+
+    file_id = open_and_parse_file(trim(filename))
+
     if (file_id==999) then
       nentries = 0
     else

diag_manager/fms_diag_yaml.F90

@@ -45,6 +45,7 @@ module fms_diag_yaml_mod
                                 fms_f2c_string
 use platform_mod, only: r4_kind, i4_kind, r8_kind, i8_kind, FMS_FILE_LEN
 use fms_mod, only: lowercase
+use fms2_io_mod, only: file_exists, get_instance_filename
 
 implicit none
 
@@ -381,10 +382,17 @@ subroutine diag_yaml_object_init(diag_subset_output)
                                            !! outputing data at every frequency)
   character(len=:), allocatable :: filename!< Diag file name (for error messages)
   logical              :: is_instantaneous !< .True. if the file is instantaneous (i.e no averaging)
+  character(len=255)   :: yamlfilename     !< Name of the expected diag_table.yaml
 
   if (diag_yaml_module_initialized) return
 
-  diag_yaml_id = open_and_parse_file("diag_table.yaml")
+  ! If doing and ensemble or nest run add the filename appendix (ens_XX or nest_XX) to the filename
+  call get_instance_filename("diag_table.yaml", yamlfilename)
+  if (index(trim(yamlfilename), "ens_") .ne. 0) then
+    if (file_exists(yamlfilename) .and. file_exists("diag_table.yaml")) &
+      call mpp_error(FATAL, "Both diag_table.yaml and "//trim(yamlfilename)//" exists, pick one!")
+  endif
+  diag_yaml_id = open_and_parse_file(trim(yamlfilename))
 
   call diag_get_value_from_key(diag_yaml_id, 0, "title", diag_yaml%diag_title)
   call get_value_from_key(diag_yaml_id, 0, "base_date", diag_yaml%diag_basedate)

field_manager/field_manager.F90

@@ -190,7 +190,7 @@ module field_manager_mod
 use    fms_mod, only : lowercase,   &
                        write_version_number, &
                        check_nml_error
-use fms2_io_mod, only: file_exists
+use fms2_io_mod, only: file_exists, get_instance_filename
 use platform_mod, only: r4_kind, r8_kind, FMS_PATH_LEN, FMS_FILE_LEN
 #ifdef use_yaml
 use fm_yaml_mod
@@ -606,18 +606,27 @@ subroutine read_field_table_yaml(nfields, table_name)
 logical                         :: fm_success !< logical for whether fm_change_list was a success
 logical                         :: subparams !< logical whether subparams exist in this iteration
 
+character(len=255) :: filename !< Name of the expected field_table.yaml
+
 if (.not.PRESENT(table_name)) then
    tbl_name = 'field_table.yaml'
 else
    tbl_name = trim(table_name)
 endif
-if (.not. file_exists(trim(tbl_name))) then
+
+call get_instance_filename(tbl_name, filename)
+if (index(trim(filename), "ens_") .ne. 0) then
+  if (file_exists(filename) .and. file_exists(tbl_name)) &
+    call mpp_error(FATAL, "Both "//trim(tbl_name)//" and "//trim(filename)//" exists, pick one!")
+endif
+
+if (.not. file_exists(trim(filename))) then
   if(present(nfields)) nfields = 0
   return
 endif
 
 ! Construct my_table object
-call build_fmTable(my_table, trim(tbl_name))
+call build_fmTable(my_table, trim(filename))
 
 do h=1,size(my_table%types)
   do i=1,size(my_table%types(h)%models)

fms2_io/fms_io_utils.F90

@@ -824,8 +824,14 @@ subroutine get_instance_filename(name_in,name_out)
      if ( i .ne. 0 ) then
         name_out = name_in(1:i-1)//'.'//trim(filename_appendix)//name_in(i:length)
      else
-     !< If .nc is not in the name, add the appendix at the end of the file
-        name_out = name_in(1:length)  //'.'//trim(filename_appendix)
+        i = index(trim(name_in), ".yaml", back=.true.)
+        if (i .ne. 0) then
+          !< If .yaml is in the filename add the appendix before it
+          name_out = name_in(1:i-1)//'.'//trim(filename_appendix)//name_in(i:length)
+        else
+          !< If .nc  and .yaml are not in the name, add the appendix at the end of the file
+          name_out = name_in(1:length)  //'.'//trim(filename_appendix)
+        endif
      end if
   end if
 

test_fms/data_override/Makefile.am

@@ -73,11 +73,11 @@ TESTS_ENVIRONMENT= test_input_path="@TEST_INPUT_PATH@" \
 # Run the test program.
 
 TESTS = test_data_override2.sh test_data_override_init.sh test_data_override2_mono.sh test_data_override2_ongrid.sh \
-        test_data_override2_scalar.sh test_data_override_weights.sh
+        test_data_override2_scalar.sh test_data_override_weights.sh test_data_override_ensembles.sh
 
 # Include these files with the distribution.
 EXTRA_DIST = test_data_override2.sh test_data_override_init.sh test_data_override2_mono.sh test_data_override2_ongrid.sh \
-             test_data_override2_scalar.sh test_data_override_weights.sh
+             test_data_override2_scalar.sh test_data_override_weights.sh test_data_override_ensembles.sh
 
 # Clean up
 CLEANFILES = input.nml *.nc* *.out diag_table data_table data_table.yaml INPUT/* *.dpi *.spi *.dyn *.spl *-files/*

test_fms/data_override/test_data_override_ensembles.sh

@@ -0,0 +1,92 @@
+#!/bin/sh
+
+#***********************************************************************
+#*                   GNU Lesser General Public License
+#*
+#* This file is part of the GFDL Flexible Modeling System (FMS).
+#*
+#* FMS is free software: you can redistribute it and/or modify it under
+#* the terms of the GNU Lesser General Public License as published by
+#* the Free Software Foundation, either version 3 of the License, or (at
+#* your option) any later version.
+#*
+#* FMS is distributed in the hope that it will be useful, but WITHOUT
+#* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#* FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+#* for more details.
+#*
+#* You should have received a copy of the GNU Lesser General Public
+#* License along with FMS.  If not, see <http://www.gnu.org/licenses/>.
+#***********************************************************************
+#
+# Copyright (c) 2019-2021 Ed Hartnett, Uriel Ramirez, Seth Underwood
+
+# Set common test settings.
+. ../test-lib.sh
+
+output_dir
+[ ! -d "INPUT" ] && mkdir -p "INPUT"
+
+cat <<_EOF > data_table.ens_01.yaml
+data_table:
+ - grid_name: OCN
+   fieldname_in_model: runoff
+   override_file:
+   - fieldname_in_file: runoff
+     file_name: INPUT/runoff.daitren.clim.1440x1080.v20180328_ens_01.nc
+     interp_method: none
+   factor: 1.0
+_EOF
+
+cat <<_EOF > data_table.ens_02.yaml
+data_table:
+ - grid_name: OCN
+   fieldname_in_model: runoff
+   override_file:
+   - fieldname_in_file: runoff
+     file_name: INPUT/runoff.daitren.clim.1440x1080.v20180328_ens_02.nc
+     interp_method: none
+   factor: 1.0
+_EOF
+
+cat <<_EOF > input.nml
+&data_override_nml
+  use_data_table_yaml = .True.
+/
+
+&test_data_override_ongrid_nml
+  test_case = 5
+/
+
+&ensemble_nml
+   ensemble_size = 2
+/
+_EOF
+
+#The test only runs with yaml
+if [ -z $parser_skip ]; then
+  for KIND in r4 r8
+  do
+    rm -rf INPUT/.
+    test_expect_success "test_data_override with two ensembles  -yaml (${KIND})" '
+      mpirun -n 12 ../test_data_override_ongrid_${KIND}
+      '
+  done
+fi
+
+cat <<_EOF > data_table.yaml
+data_table:
+ - grid_name: OCN
+   fieldname_in_model: runoff
+   override_file:
+   - fieldname_in_file: runoff
+     file_name: INPUT/runoff.daitren.clim.1440x1080.v20180328_ens_02.nc
+     interp_method: none
+   factor: 1.0
+_EOF
+
+test_expect_failure "test_data_override with both data_table.yaml and data_table.ens_xx.yaml files" '
+      mpirun -n 12 ../test_data_override_ongrid_${KIND}
+      '
+rm -rf INPUT
+test_done