From 7be0f75ad53ed079cc01ab0658437dedfd8a1928 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 25 Jan 2024 00:38:14 -0600 Subject: [PATCH 01/11] Adding cmake file for ANL workstation Adding cmake file for building SCREAM on ANL workstation --- components/eamxx/cmake/machine-files/anlgce.cmake | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 components/eamxx/cmake/machine-files/anlgce.cmake diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake new file mode 100644 index 00000000000..37570d58eec --- /dev/null +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -0,0 +1,7 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) + +# Remove this if you are using a resource manager (slurm etc) +set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") From 107da5a2ba6c6e9c077c2f0f038d61bafb0c5e3b Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Tue, 19 Mar 2024 19:40:28 -0500 Subject: [PATCH 02/11] Support for EAMXX/SCREAM testing in ANL GCE Adding support for EAMXX standalone testing on ANL GCE machine (Run EAMXX standalone tests on ANL GCE using "./scripts/test-all-scream -m anlgce ...") --- components/eamxx/cmake/machine-files/anlgce.cmake | 5 +++++ components/eamxx/scripts/machines_specs.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index 37570d58eec..079000059d1 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -2,6 +2,11 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") + +# EKAT MPI settings +set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index df89ae7b5ad..3c7f90dff3f 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -75,7 +75,10 @@ ["mpic++","mpif90","mpicc"], "srun --mpi=pmi2 -l -N 1 --kill-on-bad-exit --cpu_bind=cores", "/lcrc/group/e3sm/baselines/chrys/intel/scream"), - + "anlgce" : ([". /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/sh", "module purge", "module load autoconf/2.69-bmnwajj automake/1.16.3-r7w24o4 libtool/2.4.6-uh3mpsu m4/1.4.19-7fztfyz cmake/3.20.5-zyz2eld gcc/11.1.0-qsjmpcg zlib/1.2.11-p7dmb5p", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), From efcc246b2a6cd43f5a8fca470e5c768701413c7b Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 24 Jan 2024 13:21:51 -0600 Subject: [PATCH 03/11] Updating SCORPIO to v1.6.1 SCORPIO v1.6.0 includes the following fixes/enhancements, * Support for reading ADIOS BP files and updates to the BP to NetCDF conversion tool to integrate with CIME/E3SM * Adding a SCORPIO CMake package during install * Support for new CDF5 types in the C interface * Support for pio_set_fill() * Fix for performance issues while writing large output files (hang during enddef() call) * Multiple fixes including fix for build issues on Perlmutter SCORPIO v1.6.1 includes the following fixes, * Fixes for ADIOS to NetCDF conversion tool hang/crashes in certain scenarios --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index de0b1ca2200..16e01694a01 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit de0b1ca2200f62c6eb5e3fd40147965409e97123 +Subproject commit 16e01694a011b50bd5ff68277aa2de204dfad36d From c3cc055f39e74863ea030d848c2b8f2e1b7d8b66 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 25 Jan 2024 00:42:27 -0600 Subject: [PATCH 04/11] Option to specify I/O type for hist/rest files Adding an option to specify the SCORPIO I/O type (PnetCDF, ADIOS etc) to use for history and restart files via SCREAM YAML configuration files. The default YAML configuration files are modified to add a new parameter, "iotype", that can be used to specify the I/O types to use for history and restart files. The default is specified such that the history files are written out using PnetCDF (uses the pnetcdf iotype) while the I/O type used for restart files is specified as "default" (picked up via CIME for E3SM runs) Some function interfaces were modified to accept the new I/O type. Tests that use the modified function interfaces were updated accordingly --- .../cime_config/namelist_defaults_scream.xml | 1 + .../control/intensive_observation_period.cpp | 4 +- .../src/physics/spa/spa_functions_impl.hpp | 2 +- .../grid/remap/horiz_interp_remapper_data.cpp | 2 +- .../share/grid/remap/vertical_remapper.cpp | 2 +- .../eamxx/src/share/io/scorpio_input.cpp | 5 +- .../eamxx/src/share/io/scorpio_input.hpp | 1 + .../src/share/io/scream_io_file_specs.hpp | 2 + .../eamxx/src/share/io/scream_io_utils.hpp | 34 +++++++++++++ .../src/share/io/scream_output_manager.cpp | 10 ++-- .../src/share/io/scream_scorpio_interface.F90 | 49 ++++++++++++++----- .../src/share/io/scream_scorpio_interface.cpp | 16 +++--- .../src/share/io/scream_scorpio_interface.hpp | 2 +- .../io/scream_scorpio_interface_iso_c2f.F90 | 5 +- .../src/share/io/tests/io_remap_test.cpp | 2 +- .../share/tests/coarsening_remapper_tests.cpp | 2 +- .../tests/refining_remapper_p2p_tests.cpp | 2 +- .../share/tests/vertical_remapper_tests.cpp | 2 +- .../share/util/eamxx_time_interpolation.cpp | 2 +- .../homme_shoc_cld_p3_rrtmgp/output.yaml | 1 + 20 files changed, 110 insertions(+), 36 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 07ef2793f38..6c240f3542d 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -503,6 +503,7 @@ be lost if SCREAM_HACK_XML is not enabled. ./${CASE}.scream + default ${REST_N} ${REST_OPTION} diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index d925b21bfaa..ff2335c940b 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -45,7 +45,7 @@ void read_dimensionless_variable_from_file(const std::string& filename, int ncid, varid, err1, err2; bool was_open = scorpio::is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - scorpio::register_file(filename,scorpio::FileMode::Read); + scorpio::register_file(filename,scorpio::FileMode::Read,0); } ncid = scorpio::get_file_ncid_c2f (filename.c_str()); err1 = PIOc_inq_varid(ncid,varname.c_str(),&varid); @@ -87,7 +87,7 @@ void read_variable_from_file(const std::string& filename, } // Read into data - scorpio::register_file(filename, scorpio::FileMode::Read); + scorpio::register_file(filename, scorpio::FileMode::Read,0); std::string io_decomp_tag = varname+","+filename; scorpio::register_variable(filename, varname, varname, dimnames, vartype, io_decomp_tag); std::vector dof_offsets(data_size); diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index 748983006a1..eee2b093a2c 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -78,7 +78,7 @@ create_horiz_remapper ( { using namespace ShortFieldTagsNames; - scorpio::register_file(spa_data_file,scorpio::Read); + scorpio::register_file(spa_data_file,scorpio::Read,0); const int nlevs_data = scorpio::get_dimlen(spa_data_file,"lev"); const int ncols_data = scorpio::get_dimlen(spa_data_file,"ncol"); const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp index d44fc6aa2f0..298a156ae24 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -38,7 +38,7 @@ get_my_triplets (const std::string& map_file) const using namespace ShortFieldTagsNames; // 1. Load the map file chunking it evenly across all ranks - scorpio::register_file(map_file,scorpio::FileMode::Read); + scorpio::register_file(map_file,scorpio::FileMode::Read,0); // 1.1 Create a "helper" grid, with as many dofs as the number // of triplets in the map file, and divided linearly across ranks diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 9abefed9eed..59b1f7b3915 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -54,7 +54,7 @@ VerticalRemapper (const grid_ptr_type& src_grid, // remapping the target field will be defined on the same DOFs // as the source field, but will have a different number of // vertical levels. - scorpio::register_file(map_file,scorpio::FileMode::Read); + scorpio::register_file(map_file,scorpio::FileMode::Read,0); m_num_remap_levs = scorpio::get_dimlen(map_file,"lev"); auto tgt_grid = src_grid->clone("vertical_remap_tgt_grid",true); diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index c4f023cc396..858fcdd6b1b 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -350,7 +350,10 @@ void AtmosphereInput::finalize() /* ---------------------------------------------------------- */ void AtmosphereInput::init_scorpio_structures() { - scorpio::register_file(m_filename,scorpio::Read); + std::string iotype_str = m_params.get("iotype", "default"); + int iotype = str2iotype(iotype_str); + + scorpio::register_file(m_filename,scorpio::Read,iotype); // Register variables with netCDF file. register_variables(); diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 811d5f6f771..4655b58777e 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -2,6 +2,7 @@ #define SCREAM_SCORPIO_INPUT_HPP #include "share/io/scream_scorpio_interface.hpp" +#include "share/io/scream_io_utils.hpp" #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" #include "share/grid/grids_manager.hpp" diff --git a/components/eamxx/src/share/io/scream_io_file_specs.hpp b/components/eamxx/src/share/io/scream_io_file_specs.hpp index 0e9ab2e042a..c80db36e094 100644 --- a/components/eamxx/src/share/io/scream_io_file_specs.hpp +++ b/components/eamxx/src/share/io/scream_io_file_specs.hpp @@ -80,6 +80,8 @@ struct IOFileSpecs { bool is_open = false; std::string filename; + int iotype = 0; + // If positive, flush the output file every these many snapshots int flush_frequency = std::numeric_limits::max(); diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 8cbcd39a1a2..13417bfaec3 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -59,6 +59,40 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } +inline int str2iotype(const std::string &str) +{ + if(str == "default"){ + return 0; + } + else if(str == "netcdf"){ + return 1; + } + else if(str == "pnetcdf"){ + return 2; + } + else if(str == "adios"){ + return 3; + } + else if(str == "hdf5"){ + return 4; + } + else{ + return 0; + } +} + +inline std::string iotype2str(int iotype) +{ + switch(iotype){ + case 0: return "default"; + case 1: return "netcdf"; + case 2: return "pnetcdf"; + case 3: return "adios"; + case 4: return "hdf5"; + default: return "default"; + } +} + std::string find_filename_in_rpointer ( const std::string& casename, const bool model_restart, diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index e9f8f588702..fc438ea7483 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -237,8 +237,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const auto& last_output_filename = get_attribute(rhist_file,"last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { - - scorpio::register_file(last_output_filename,scorpio::Read); + scorpio::register_file(last_output_filename,scorpio::Read,m_output_file_specs.iotype); int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); scorpio::eam_pio_closefile(last_output_filename); @@ -680,6 +679,11 @@ set_params (const ekat::ParameterList& params, m_checkpoint_file_specs.ftype = FileType::HistoryRestart; } } + + // Set the iotype to use for the output file + std::string iotype = m_params.get("iotype", "default"); + m_output_file_specs.iotype = str2iotype(iotype); + m_checkpoint_file_specs.iotype = str2iotype(iotype); } /*===============================================================================================*/ @@ -698,7 +702,7 @@ setup_file ( IOFileSpecs& filespecs, const auto& filename = filespecs.filename; // Register new netCDF file for output. Check if we need to append to an existing file auto mode = m_resume_output_file ? Append : Write; - register_file(filename,mode); + register_file(filename,mode,filespecs.iotype); if (m_resume_output_file) { eam_pio_redef(filename); } diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.F90 b/components/eamxx/src/share/io/scream_scorpio_interface.F90 index 50401f7f305..4bb9c7efa54 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface.F90 @@ -45,7 +45,8 @@ module scream_scorpio_interface !------------ use pio_types, only: iosystem_desc_t, file_desc_t, var_desc_t, io_desc_t, & pio_noerr, pio_global, & - PIO_int, PIO_real, PIO_double, PIO_float=>PIO_real + PIO_int, PIO_real, PIO_double, PIO_float=>PIO_real,& + pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_adios use pio_kinds, only: PIO_OFFSET_KIND use mpi, only: mpi_abort, mpi_comm_size, mpi_comm_rank @@ -79,7 +80,7 @@ module scream_scorpio_interface eam_update_time, & ! Update the timestamp (i.e. time variable) for a given pio netCDF file read_time_at_index ! Returns the time stamp for a specific time index - private :: errorHandle, get_coord, is_read, is_write, is_append + private :: errorHandle, get_coord, is_read, is_write, is_append, scream_iotype_to_pio_iotype ! Universal PIO variables for the module integer :: atm_mpicom @@ -193,10 +194,11 @@ module scream_scorpio_interface !=====================================================================! ! Register a PIO file to be used for input/output operations. ! If file is already open, ensures file_purpose matches the current one - subroutine register_file(filename,file_purpose) + subroutine register_file(filename,file_purpose,iotype) character(len=*), intent(in) :: filename integer, intent(in) :: file_purpose + integer, intent(in) :: iotype type(pio_atm_file_t), pointer :: pio_file @@ -204,7 +206,7 @@ subroutine register_file(filename,file_purpose) call errorHandle("PIO ERROR: local pio_subsystem pointer has not been established yet.",-999) endif - call get_pio_atm_file(filename,pio_file,file_purpose) + call get_pio_atm_file(filename,pio_file,file_purpose,iotype) end subroutine register_file !=====================================================================! ! Mandatory call to finish the variable and dimension definition phase @@ -867,26 +869,46 @@ function is_eam_pio_subsystem_inited() result(is_it) bind(c) is_it = LOGICAL(associated(pio_subsystem),kind=c_bool) end function is_eam_pio_subsystem_inited +!=====================================================================! + function scream_iotype_to_pio_iotype(siotype) result(piotype) + integer, intent(in) :: siotype + integer :: piotype + + if(siotype == 0) then + piotype = pio_iotype + else if(siotype == 1) then + piotype = pio_iotype_netcdf + else if(siotype == 2) then + piotype = pio_iotype_pnetcdf + else if(siotype == 3) then + piotype = pio_iotype_adios + else + piotype = pio_iotype + end if + end function scream_iotype_to_pio_iotype !=====================================================================! ! Create a pio netCDF file with the appropriate name. - subroutine eam_pio_createfile(File,fname) + subroutine eam_pio_createfile(File,fname,iotype) use pio, only: pio_createfile use pio_types, only: pio_clobber type(file_desc_t), intent(inout) :: File ! Pio file Handle character(len=*), intent(in) :: fname ! Pio file name + integer, intent(in) :: iotype !-- integer :: retval ! PIO error return value integer :: mode ! Mode for how to handle the new file + integer :: piotype mode = ior(pio_mode,pio_clobber) ! Set to CLOBBER for now, TODO: fix to allow for optional mode type like in CAM - retval = pio_createfile(pio_subsystem,File,pio_iotype,fname,mode) + piotype = scream_iotype_to_pio_iotype(iotype) + retval = pio_createfile(pio_subsystem,File,piotype,fname,mode) call errorHandle("PIO ERROR: unable to create file: "//trim(fname),retval) end subroutine eam_pio_createfile !=====================================================================! ! Open an already existing netCDF file. - subroutine eam_pio_openfile(pio_file,fname) + subroutine eam_pio_openfile(pio_file,fname,iotype) use pio, only: pio_openfile use pio_types, only: pio_write, pio_nowrite @@ -895,13 +917,17 @@ subroutine eam_pio_openfile(pio_file,fname) !-- integer :: retval ! PIO error return value integer :: mode ! Mode for how to handle the new file + integer, intent(in) :: iotype + + integer :: piotype if (is_read(pio_file%purpose)) then mode = pio_nowrite else mode = pio_write endif - retval = pio_openfile(pio_subsystem,pio_file%pioFileDesc,pio_iotype,fname,mode) + piotype = scream_iotype_to_pio_iotype(iotype) + retval = pio_openfile(pio_subsystem,pio_file%pioFileDesc,piotype,fname,mode) call errorHandle("PIO ERROR: unable to open file: "//trim(fname),retval) if (is_append(pio_file%purpose)) then @@ -1361,12 +1387,13 @@ subroutine lookup_pio_atm_file(filename,pio_file,found,pio_file_list_ptr_in) end subroutine lookup_pio_atm_file !=====================================================================! ! Create a new pio file pointer based on filename. - subroutine get_pio_atm_file(filename,pio_file,purpose) + subroutine get_pio_atm_file(filename,pio_file,purpose,iotype) use pio, only: PIO_inq_dimid, PIO_inq_dimlen character(len=*),intent(in) :: filename ! Name of file to be found type(pio_atm_file_t), pointer :: pio_file ! Pointer to pio_atm_output structure associated with this filename integer,intent(in) :: purpose ! Purpose for this file lookup, 0 = find already existing, 1 = create new as output, 2 = open new as input + integer,intent(in) :: iotype logical :: found type(pio_file_list_t), pointer :: new_list_item @@ -1404,7 +1431,7 @@ subroutine get_pio_atm_file(filename,pio_file,purpose) pio_file%purpose = purpose if (is_read(purpose) .or. is_append(purpose)) then ! Either read or append to existing file. Either way, file must exist on disk - call eam_pio_openfile(pio_file,trim(pio_file%filename)) + call eam_pio_openfile(pio_file,trim(pio_file%filename),iotype) ! Update the numRecs to match the number of recs in this file. ierr = pio_inq_dimid(pio_file%pioFileDesc,"time",time_id) if (ierr.ne.0) then @@ -1417,7 +1444,7 @@ subroutine get_pio_atm_file(filename,pio_file,purpose) end if elseif (is_write(purpose)) then ! New output file - call eam_pio_createfile(pio_file%pioFileDesc,trim(pio_file%filename)) + call eam_pio_createfile(pio_file%pioFileDesc,trim(pio_file%filename),iotype) call eam_pio_createHeader(pio_file%pioFileDesc) else call errorHandle("PIO Error: get_pio_atm_file with filename = "//trim(filename)//", purpose (int) assigned to this lookup is not valid" ,-999) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index a74ceecd393..9c193d8ae4a 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -15,7 +15,7 @@ using scream::Int; extern "C" { // Fortran routines to be called from C++ - void register_file_c2f(const char*&& filename, const int& mode); + void register_file_c2f(const char*&& filename, const int& mode, const int& iotype); int get_file_mode_c2f(const char*&& filename); void set_decomp_c2f(const char*&& filename); void set_dof_c2f(const char*&& filename,const char*&& varname,const Int dof_len,const std::int64_t *x_dof); @@ -94,8 +94,8 @@ void eam_pio_finalize() { eam_pio_finalize_c2f(); } /* ----------------------------------------------------------------- */ -void register_file(const std::string& filename, const FileMode mode) { - register_file_c2f(filename.c_str(),mode); +void register_file(const std::string& filename, const FileMode mode, const int iotype) { + register_file_c2f(filename.c_str(),mode,iotype); } /* ----------------------------------------------------------------- */ void eam_pio_closefile(const std::string& filename) { @@ -118,7 +118,7 @@ int get_dimlen(const std::string& filename, const std::string& dimname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read); + register_file(filename,Read,0); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -154,7 +154,7 @@ bool has_dim (const std::string& filename, const std::string& dimname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read); + register_file(filename,Read,0); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -181,7 +181,7 @@ bool has_variable (const std::string& filename, const std::string& varname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read); + register_file(filename,Read,0); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -212,7 +212,7 @@ bool has_attribute (const std::string& filename, const std::string& varname, con bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read); + register_file(filename,Read,0); } // Get file id @@ -513,7 +513,7 @@ ekat::any get_any_attribute (const std::string& filename, const std::string& att } /* ----------------------------------------------------------------- */ ekat::any get_any_attribute (const std::string& filename, const std::string& var_name, const std::string& att_name) { - register_file(filename,Read); + register_file(filename,Read,0); auto ncid = get_file_ncid_c2f (filename.c_str()); EKAT_REQUIRE_MSG (ncid>=0, "[get_any_attribute] Error! Could not retrieve file ncid.\n" diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index b64ca3ef25f..acaf587d447 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -33,7 +33,7 @@ namespace scorpio { void eam_pio_closefile(const std::string& filename); void eam_flush_file(const std::string& filename); /* Register a new file to be used for input/output with the scorpio module */ - void register_file(const std::string& filename, const FileMode mode); + void register_file(const std::string& filename, const FileMode mode, int iotype); /* Sets the IO decompostion for all variables in a particular filename. Required after all variables have been registered. Called once per file. */ int get_dimlen(const std::string& filename, const std::string& dimname); bool has_dim(const std::string& filename, const std::string& dimname); diff --git a/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 b/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 index 9e50842b245..2f1263beef7 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface_iso_c2f.F90 @@ -76,15 +76,16 @@ function is_file_open_c2f(filename_in,purpose) result(res) bind(c) endif end function is_file_open_c2f !=====================================================================! - subroutine register_file_c2f(filename_in,purpose) bind(c) + subroutine register_file_c2f(filename_in,purpose,iotype) bind(c) use scream_scorpio_interface, only : register_file type(c_ptr), intent(in) :: filename_in integer(kind=c_int), intent(in) :: purpose + integer(kind=c_int), intent(in) :: iotype character(len=256) :: filename call convert_c_string(filename_in,filename) - call register_file(trim(filename),purpose) + call register_file(trim(filename),purpose,iotype) end subroutine register_file_c2f !=====================================================================! diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index bfed937636e..2d8c0aa2aef 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -106,7 +106,7 @@ TEST_CASE("io_remap_test","io_remap_test") // Write remap data to file const std::string remap_filename = "remap_weights_np"+std::to_string(io_comm.size())+".nc"; - scorpio::register_file(remap_filename, scorpio::FileMode::Write); + scorpio::register_file(remap_filename, scorpio::FileMode::Write,0); scorpio::register_dimension(remap_filename,"n_a", "n_a", ncols_src, true); scorpio::register_dimension(remap_filename,"n_b", "n_b", ncols_tgt, true); diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index 13832c49044..f7951a48445 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -235,7 +235,7 @@ void create_remap_file(const std::string& filename, const int ngdofs_tgt) const int ngdofs_src = ngdofs_tgt + 1; const int nnz = 2*ngdofs_tgt; - scorpio::register_file(filename, scorpio::FileMode::Write); + scorpio::register_file(filename, scorpio::FileMode::Write,0); scorpio::register_dimension(filename,"n_a", "n_a", ngdofs_src, true); scorpio::register_dimension(filename,"n_b", "n_b", ngdofs_tgt, true); diff --git a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp index 00ee58ef503..0fc4d05a8b8 100644 --- a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp +++ b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp @@ -122,7 +122,7 @@ void write_map_file (const std::string& filename, const int ngdofs_src) { // Existing dofs are "copied", added dofs are averaged from neighbors const int nnz = ngdofs_src + 2*(ngdofs_src-1); - scorpio::register_file(filename, scorpio::FileMode::Write); + scorpio::register_file(filename, scorpio::FileMode::Write,0); scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index 6ec4a40c123..db05558aa54 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -97,7 +97,7 @@ Real data_func(const int col, const int vec, const Real pres) { void create_remap_file(const std::string& filename, const int nlevs, const std::vector& dofs_p, const std::vector& p_tgt) { - scorpio::register_file(filename, scorpio::FileMode::Write); + scorpio::register_file(filename, scorpio::FileMode::Write,0); scorpio::register_dimension(filename,"lev","lev",nlevs, false); diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index 3515bcdaac9..c8cdccbc914 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -273,7 +273,7 @@ void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { if (ii==0) { ts_ref = ts_file_start; } - scorpio::register_file(filename,scorpio::Read); + scorpio::register_file(filename,scorpio::Read,0); const int ntime = scorpio::get_dimlen(filename,"time"); for (int tt=0; tt Date: Thu, 1 Feb 2024 15:08:15 -0600 Subject: [PATCH 05/11] Adding ADIOS2 build for hipcc on Frontier Adding path to new ADIOS2 build with hipcc for Frontier. Also see E3SM-Project/scorpio#528 --- cime_config/machines/config_machines.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 0e2915a8c19..b0e083c6f51 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1472,6 +1472,10 @@ spread threads + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + From ceb43365edb1677b503186e06196c0e2636c0cde Mon Sep 17 00:00:00 2001 From: Jayesh Krishna Date: Thu, 7 Mar 2024 18:02:17 -0500 Subject: [PATCH 06/11] Reset MPICH GPU support for conversion tool Disable MPICH/MPI GPU support for ADIOS to NetCDF conversion tool. Without this change the conversion tool crashes (with message "MPIDI_CRAY_init: GPU_SUPPORT_ENABLED is requested, but GTL library is not linked" - most likely due to the way SCREAM builds/links SCORPIO) on Frontier. --- cime_config/customize/case_post_run_io.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cime_config/customize/case_post_run_io.py b/cime_config/customize/case_post_run_io.py index bb44477ebb5..83419aa8134 100755 --- a/cime_config/customize/case_post_run_io.py +++ b/cime_config/customize/case_post_run_io.py @@ -74,6 +74,12 @@ def _convert_adios_to_nc(case): # Load the environment case.load_env(reset=True) + # Reset MPICH/MPI GPU support, if enabled + is_mpich_gpu_enabled = os.environ.get('MPICH_GPU_SUPPORT_ENABLED') + if int(0 if is_mpich_gpu_enabled is None else is_mpich_gpu_enabled) == 1: + logger.info("Resetting support for GPU in MPICH/MPI library (since its not used by the tool)") + os.environ['MPICH_GPU_SUPPORT_ENABLED'] = str(0); + run_func = lambda: run_cmd(cmd, from_dir=rundir)[0] # Run the modified case From 282405d8846bacf668da8d86c0c9bee118a41626 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Mon, 11 Mar 2024 13:36:08 -0500 Subject: [PATCH 07/11] Moving SPIO I/O type utils inside scorpio nspace Moving SCORPIO I/O type utils inside the SCORPIO namespace. Also adding a new enum for all supported SCORPIO I/O types. --- .../control/intensive_observation_period.cpp | 4 +- .../src/physics/spa/spa_functions_impl.hpp | 2 +- .../grid/remap/horiz_interp_remapper_data.cpp | 2 +- .../share/grid/remap/vertical_remapper.cpp | 2 +- .../eamxx/src/share/io/scorpio_input.cpp | 2 +- .../eamxx/src/share/io/scorpio_input.hpp | 1 - .../eamxx/src/share/io/scream_io_utils.hpp | 34 -------------- .../src/share/io/scream_output_manager.cpp | 4 +- .../src/share/io/scream_scorpio_interface.cpp | 10 ++-- .../src/share/io/scream_scorpio_interface.hpp | 47 ++++++++++++++++++- .../src/share/io/tests/io_remap_test.cpp | 2 +- .../share/tests/coarsening_remapper_tests.cpp | 2 +- .../tests/refining_remapper_p2p_tests.cpp | 2 +- .../share/tests/vertical_remapper_tests.cpp | 2 +- .../share/util/eamxx_time_interpolation.cpp | 2 +- 15 files changed, 64 insertions(+), 54 deletions(-) diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index ff2335c940b..d925b21bfaa 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -45,7 +45,7 @@ void read_dimensionless_variable_from_file(const std::string& filename, int ncid, varid, err1, err2; bool was_open = scorpio::is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - scorpio::register_file(filename,scorpio::FileMode::Read,0); + scorpio::register_file(filename,scorpio::FileMode::Read); } ncid = scorpio::get_file_ncid_c2f (filename.c_str()); err1 = PIOc_inq_varid(ncid,varname.c_str(),&varid); @@ -87,7 +87,7 @@ void read_variable_from_file(const std::string& filename, } // Read into data - scorpio::register_file(filename, scorpio::FileMode::Read,0); + scorpio::register_file(filename, scorpio::FileMode::Read); std::string io_decomp_tag = varname+","+filename; scorpio::register_variable(filename, varname, varname, dimnames, vartype, io_decomp_tag); std::vector dof_offsets(data_size); diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index eee2b093a2c..748983006a1 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -78,7 +78,7 @@ create_horiz_remapper ( { using namespace ShortFieldTagsNames; - scorpio::register_file(spa_data_file,scorpio::Read,0); + scorpio::register_file(spa_data_file,scorpio::Read); const int nlevs_data = scorpio::get_dimlen(spa_data_file,"lev"); const int ncols_data = scorpio::get_dimlen(spa_data_file,"ncol"); const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp index 298a156ae24..d44fc6aa2f0 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -38,7 +38,7 @@ get_my_triplets (const std::string& map_file) const using namespace ShortFieldTagsNames; // 1. Load the map file chunking it evenly across all ranks - scorpio::register_file(map_file,scorpio::FileMode::Read,0); + scorpio::register_file(map_file,scorpio::FileMode::Read); // 1.1 Create a "helper" grid, with as many dofs as the number // of triplets in the map file, and divided linearly across ranks diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 59b1f7b3915..9abefed9eed 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -54,7 +54,7 @@ VerticalRemapper (const grid_ptr_type& src_grid, // remapping the target field will be defined on the same DOFs // as the source field, but will have a different number of // vertical levels. - scorpio::register_file(map_file,scorpio::FileMode::Read,0); + scorpio::register_file(map_file,scorpio::FileMode::Read); m_num_remap_levs = scorpio::get_dimlen(map_file,"lev"); auto tgt_grid = src_grid->clone("vertical_remap_tgt_grid",true); diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 858fcdd6b1b..af088658869 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -351,7 +351,7 @@ void AtmosphereInput::finalize() void AtmosphereInput::init_scorpio_structures() { std::string iotype_str = m_params.get("iotype", "default"); - int iotype = str2iotype(iotype_str); + int iotype = scorpio::str2iotype(iotype_str); scorpio::register_file(m_filename,scorpio::Read,iotype); diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 4655b58777e..811d5f6f771 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -2,7 +2,6 @@ #define SCREAM_SCORPIO_INPUT_HPP #include "share/io/scream_scorpio_interface.hpp" -#include "share/io/scream_io_utils.hpp" #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" #include "share/grid/grids_manager.hpp" diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 13417bfaec3..8cbcd39a1a2 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -59,40 +59,6 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } -inline int str2iotype(const std::string &str) -{ - if(str == "default"){ - return 0; - } - else if(str == "netcdf"){ - return 1; - } - else if(str == "pnetcdf"){ - return 2; - } - else if(str == "adios"){ - return 3; - } - else if(str == "hdf5"){ - return 4; - } - else{ - return 0; - } -} - -inline std::string iotype2str(int iotype) -{ - switch(iotype){ - case 0: return "default"; - case 1: return "netcdf"; - case 2: return "pnetcdf"; - case 3: return "adios"; - case 4: return "hdf5"; - default: return "default"; - } -} - std::string find_filename_in_rpointer ( const std::string& casename, const bool model_restart, diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index fc438ea7483..2458a4e95c8 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -682,8 +682,8 @@ set_params (const ekat::ParameterList& params, // Set the iotype to use for the output file std::string iotype = m_params.get("iotype", "default"); - m_output_file_specs.iotype = str2iotype(iotype); - m_checkpoint_file_specs.iotype = str2iotype(iotype); + m_output_file_specs.iotype = scorpio::str2iotype(iotype); + m_checkpoint_file_specs.iotype = scorpio::str2iotype(iotype); } /*===============================================================================================*/ diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 9c193d8ae4a..ead1aa0fbd6 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -118,7 +118,7 @@ int get_dimlen(const std::string& filename, const std::string& dimname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read,0); + register_file(filename,Read); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -154,7 +154,7 @@ bool has_dim (const std::string& filename, const std::string& dimname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read,0); + register_file(filename,Read); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -181,7 +181,7 @@ bool has_variable (const std::string& filename, const std::string& varname) bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read,0); + register_file(filename,Read); } ncid = get_file_ncid_c2f (filename.c_str()); @@ -212,7 +212,7 @@ bool has_attribute (const std::string& filename, const std::string& varname, con bool was_open = is_file_open_c2f(filename.c_str(),-1); if (not was_open) { - register_file(filename,Read,0); + register_file(filename,Read); } // Get file id @@ -513,7 +513,7 @@ ekat::any get_any_attribute (const std::string& filename, const std::string& att } /* ----------------------------------------------------------------- */ ekat::any get_any_attribute (const std::string& filename, const std::string& var_name, const std::string& att_name) { - register_file(filename,Read,0); + register_file(filename,Read); auto ncid = get_file_ncid_c2f (filename.c_str()); EKAT_REQUIRE_MSG (ncid>=0, "[get_any_attribute] Error! Could not retrieve file ncid.\n" diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index acaf587d447..c5ae69ea6c6 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -24,6 +24,51 @@ namespace scorpio { Append = 2, Write = 4 }; + + // I/O types supported + enum IOType { + // Default I/O type is used to let the code choose I/O type as needed (via CIME) + DefaultIOType = 0, + NetCDF, + PnetCDF, + Adios, + Hdf5 + }; + + inline int str2iotype(const std::string &str) + { + if(str == "default"){ + return static_cast(IOType::DefaultIOType); + } + else if(str == "netcdf"){ + return static_cast(IOType::NetCDF); + } + else if(str == "pnetcdf"){ + return static_cast(IOType::PnetCDF); + } + else if(str == "adios"){ + return static_cast(IOType::Adios); + } + else if(str == "hdf5"){ + return static_cast(IOType::Hdf5); + } + else{ + return static_cast(IOType::DefaultIOType); + } + } + + inline std::string iotype2str(int iotype) + { + switch(iotype){ + case static_cast(IOType::DefaultIOType): return "default"; + case static_cast(IOType::NetCDF): return "netcdf"; + case static_cast(IOType::PnetCDF): return "pnetcdf"; + case static_cast(IOType::Adios): return "adios"; + case static_cast(IOType::Hdf5): return "hdf5"; + default: return "default"; + } + } + /* All scorpio usage requires that the pio_subsystem is initialized. Happens only once per simulation */ void eam_init_pio_subsystem(const ekat::Comm& comm); void eam_init_pio_subsystem(const int mpicom, const int atm_id = 0); @@ -33,7 +78,7 @@ namespace scorpio { void eam_pio_closefile(const std::string& filename); void eam_flush_file(const std::string& filename); /* Register a new file to be used for input/output with the scorpio module */ - void register_file(const std::string& filename, const FileMode mode, int iotype); + void register_file(const std::string& filename, const FileMode mode, int iotype = IOType::DefaultIOType); /* Sets the IO decompostion for all variables in a particular filename. Required after all variables have been registered. Called once per file. */ int get_dimlen(const std::string& filename, const std::string& dimname); bool has_dim(const std::string& filename, const std::string& dimname); diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index 2d8c0aa2aef..bfed937636e 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -106,7 +106,7 @@ TEST_CASE("io_remap_test","io_remap_test") // Write remap data to file const std::string remap_filename = "remap_weights_np"+std::to_string(io_comm.size())+".nc"; - scorpio::register_file(remap_filename, scorpio::FileMode::Write,0); + scorpio::register_file(remap_filename, scorpio::FileMode::Write); scorpio::register_dimension(remap_filename,"n_a", "n_a", ncols_src, true); scorpio::register_dimension(remap_filename,"n_b", "n_b", ncols_tgt, true); diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index f7951a48445..13832c49044 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -235,7 +235,7 @@ void create_remap_file(const std::string& filename, const int ngdofs_tgt) const int ngdofs_src = ngdofs_tgt + 1; const int nnz = 2*ngdofs_tgt; - scorpio::register_file(filename, scorpio::FileMode::Write,0); + scorpio::register_file(filename, scorpio::FileMode::Write); scorpio::register_dimension(filename,"n_a", "n_a", ngdofs_src, true); scorpio::register_dimension(filename,"n_b", "n_b", ngdofs_tgt, true); diff --git a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp index 0fc4d05a8b8..00ee58ef503 100644 --- a/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp +++ b/components/eamxx/src/share/tests/refining_remapper_p2p_tests.cpp @@ -122,7 +122,7 @@ void write_map_file (const std::string& filename, const int ngdofs_src) { // Existing dofs are "copied", added dofs are averaged from neighbors const int nnz = ngdofs_src + 2*(ngdofs_src-1); - scorpio::register_file(filename, scorpio::FileMode::Write,0); + scorpio::register_file(filename, scorpio::FileMode::Write); scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); diff --git a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp index db05558aa54..6ec4a40c123 100644 --- a/components/eamxx/src/share/tests/vertical_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/vertical_remapper_tests.cpp @@ -97,7 +97,7 @@ Real data_func(const int col, const int vec, const Real pres) { void create_remap_file(const std::string& filename, const int nlevs, const std::vector& dofs_p, const std::vector& p_tgt) { - scorpio::register_file(filename, scorpio::FileMode::Write,0); + scorpio::register_file(filename, scorpio::FileMode::Write); scorpio::register_dimension(filename,"lev","lev",nlevs, false); diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index c8cdccbc914..3515bcdaac9 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -273,7 +273,7 @@ void TimeInterpolation::set_file_data_triplets(const vos_type& list_of_files) { if (ii==0) { ts_ref = ts_file_start; } - scorpio::register_file(filename,scorpio::Read,0); + scorpio::register_file(filename,scorpio::Read); const int ntime = scorpio::get_dimlen(filename,"time"); for (int tt=0; tt Date: Mon, 11 Mar 2024 21:19:38 -0500 Subject: [PATCH 08/11] Disable BP to NetCDF conversion Disable BP to NetCDF conversion by explicitly commenting it out from the workflow --- cime_config/machines/config_workflow.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cime_config/machines/config_workflow.xml b/cime_config/machines/config_workflow.xml index 5dd81437a35..b2eae349aff 100644 --- a/cime_config/machines/config_workflow.xml +++ b/cime_config/machines/config_workflow.xml @@ -37,7 +37,7 @@ $BUILD_COMPLETE and $TEST - + - (case.run and case.post_run_io) or case.test + + case.run or case.test $DOUT_S 1 From 67aff454baed7280a7c05be26738a0ec456ea63a Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Tue, 19 Mar 2024 19:44:06 -0500 Subject: [PATCH 09/11] Add SCORPIO and GPTL includes Explicitly add SCORPIO and GPTL includes for EAMXX standalone builds. Note: Older versions (<= 1.4.*) of SCORPIO exported these internal paths by default. Newer versions of SCORPIO do not expose these paths, so standalone builds of EAMXX need to explicitly add them. --- components/eamxx/cmake/tpls/Scorpio.cmake | 11 +++++++++++ components/eamxx/src/share/CMakeLists.txt | 5 +++++ components/eamxx/src/share/io/CMakeLists.txt | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/components/eamxx/cmake/tpls/Scorpio.cmake b/components/eamxx/cmake/tpls/Scorpio.cmake index db174629841..b75742fe119 100644 --- a/components/eamxx/cmake/tpls/Scorpio.cmake +++ b/components/eamxx/cmake/tpls/Scorpio.cmake @@ -42,6 +42,17 @@ macro (CreateScorpioTargets) option (PIO_ENABLE_FORTRAN "Enable the Fortran library builds" ON) add_subdirectory (${E3SM_EXTERNALS_DIR}/scorpio ${CMAKE_BINARY_DIR}/externals/scorpio) + + set (SCORPIO_Fortran_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/externals/scorpio/src/flib CACHE INTERNAL "SCORPIO Fortran include dirs") + set (C_INCLUDE_DIRS "${E3SM_EXTERNALS_DIR}/scorpio/src/clib") + list(APPEND C_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/externals/scorpio/src/clib") + set (SCORPIO_C_INCLUDE_DIRS "${C_INCLUDE_DIRS}" CACHE INTERNAL "SCORPIO C include dirs") + + # Add GPTL from SCORPIO + if (NOT GPTL_PATH) + set (GPTL_PATH ${E3SM_EXTERNALS_DIR}/scorpio/src/gptl CACHE INTERNAL "Path to GPTL library") + endif () + EkatDisableAllWarning(pioc) EkatDisableAllWarning(piof) EkatDisableAllWarning(gptl) diff --git a/components/eamxx/src/share/CMakeLists.txt b/components/eamxx/src/share/CMakeLists.txt index 418d146d050..701cf89bb03 100644 --- a/components/eamxx/src/share/CMakeLists.txt +++ b/components/eamxx/src/share/CMakeLists.txt @@ -57,6 +57,11 @@ target_include_directories(scream_share PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/modules ) + +if (GPTL_PATH) + target_include_directories(scream_share PUBLIC ${GPTL_PATH}) +endif () + target_link_libraries(scream_share PUBLIC ekat pioc) target_compile_options(scream_share PUBLIC $<$:${SCREAM_Fortran_FLAGS}> diff --git a/components/eamxx/src/share/io/CMakeLists.txt b/components/eamxx/src/share/io/CMakeLists.txt index 7f9b64a829c..8866e294495 100644 --- a/components/eamxx/src/share/io/CMakeLists.txt +++ b/components/eamxx/src/share/io/CMakeLists.txt @@ -19,6 +19,14 @@ if (DEFINED ENV{ADIOS2_ROOT}) target_include_directories(scream_io PRIVATE $ENV{ADIOS2_ROOT}/include) endif () +if (SCORPIO_Fortran_INCLUDE_DIRS) + target_include_directories(scream_io PUBLIC ${SCORPIO_Fortran_INCLUDE_DIRS}) +endif () + +if (SCORPIO_C_INCLUDE_DIRS) + target_include_directories(scream_io PUBLIC ${SCORPIO_C_INCLUDE_DIRS}) +endif () + target_link_libraries(scream_io PUBLIC scream_share piof pioc) if (SCREAM_CIME_BUILD) From fa6c6e8794e383d917bf8ea0fceeec40ddcaeae8 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Thu, 21 Mar 2024 09:29:44 -0500 Subject: [PATCH 10/11] Updating SCORPIO to v1.6.2 alpha SCORPIO v1.6.2 includes the following fixes, * Support for standalone EAMXX/SCREAM builds * Fixes for ultra high res simulations using the SUBSET rearranger --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index 16e01694a01..5f2cd4b672b 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit 16e01694a011b50bd5ff68277aa2de204dfad36d +Subproject commit 5f2cd4b672bf96e09d819712e17003de3a505e9c From 6b407d0cb6cada94d2a004adbf21ce5a3baeaa97 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Wed, 27 Mar 2024 18:57:29 -0500 Subject: [PATCH 11/11] Ensure PIO calls are not optimized out At least on some machines (mappy + gcc 9.2.0) if the return value (ierr) from the PIO calls is ignored the PIO calls are optimized out (values/attributes not written out to the file etc). So adding a call to handle/touch the return value from the PIO calls to ensure that values/attributes are updated in the output file Thanks lbertag@sandia.gov for the fix. --- .../eamxx/src/share/io/scream_scorpio_interface.F90 | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.F90 b/components/eamxx/src/share/io/scream_scorpio_interface.F90 index 4bb9c7efa54..bfd9ad68f2d 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface.F90 @@ -446,9 +446,11 @@ subroutine register_variable(filename,shortname,longname,units, & if (is_write(pio_atm_file%purpose)) then ierr = PIO_def_var(pio_atm_file%pioFileDesc, trim(shortname), hist_var%nc_dtype, hist_var%dimid(:numdims), hist_var%piovar) - call errorHandle("PIO ERROR: could not define variable "//trim(shortname),ierr) + call errorHandle("PIO ERROR: could not define variable "//trim(shortname)//" in file "//trim(filename),ierr) ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'units', hist_var%units ) + call errorHandle("PIO ERROR: could not set attribute 'units' for variable "//trim(shortname)//" in file "//trim(filename),ierr) ierr=PIO_put_att(pio_atm_file%pioFileDesc, hist_var%piovar, 'long_name', hist_var%long_name ) + call errorHandle("PIO ERROR: could not set attribute 'long_name' for variable "//trim(shortname)//" in file "//trim(filename),ierr) else ierr = PIO_inq_varid(pio_atm_file%pioFileDesc,trim(shortname),hist_var%piovar) call errorHandle("PIO ERROR: could not retrieve id for variable "//trim(shortname)//" from file "//trim(filename),ierr) @@ -778,7 +780,10 @@ subroutine eam_update_time(filename,time) pio_atm_file%numRecs = pio_atm_file%numRecs + 1 call get_var(pio_atm_file,'time',var) ! Only update time on the file if a valid time is provided - if (time>=0) ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,(/ pio_atm_file%numRecs /), (/ 1 /), (/ time /)) + if (time>=0) then + ierr = pio_put_var(pio_atm_file%pioFileDesc,var%piovar,(/ pio_atm_file%numRecs /), (/ 1 /), (/ time /)) + call errorHandle("PIO ERROR: something went wrong while writing time var on file="//trim(filename),ierr) + endif end subroutine eam_update_time !=====================================================================! ! Assign institutions to header metadata for a specific pio output file. @@ -1150,6 +1155,7 @@ subroutine eam_pio_finalize() #if !defined(SCREAM_CIME_BUILD) call PIO_finalize(pio_subsystem, ierr) + call errorHandle("PIO ERROR: something went wrong when calling PIO_finalize.",ierr) nullify(pio_subsystem) #endif @@ -1491,6 +1497,7 @@ function read_time_at_index(filename,time_index) result(val) ierr = pio_inq_dimid(pio_atm_file%pioFileDesc,trim("time"),dim_id) call errorHandle("read_time_at_index ERROR: dimension 'time' not found in file "//trim(filename)//".",ierr) ierr = pio_inq_dimlen(pio_atm_file%pioFileDesc,dim_id,time_len) + call errorHandle("PIO Error! Something went wrong when inquiring time dim length on file file "//trim(filename)//".",ierr) if (present(time_index)) then timeidx = time_index