From 0ae7719dce253f8bf057be0b463487cc64062536 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 21 Feb 2024 14:31:14 -0800 Subject: [PATCH 01/65] Adds surface and online emission interface and a test --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- ...and_online_emissions_process_interface.cpp | 159 ++++++++++++++++++ ...and_online_emissions_process_interface.hpp | 62 +++++++ .../eamxx/src/physics/register_physics.hpp | 2 + 4 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index c5feb52e12a..9138bba6d5a 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -45,7 +45,8 @@ add_library(mam eamxx_mam_optics_process_interface.cpp eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp - eamxx_mam_wetscav_process_interface.cpp) + eamxx_mam_wetscav_process_interface.cpp + eamxx_mam_srf_and_online_emissions_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp new file mode 100644 index 00000000000..c107dbe6970 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -0,0 +1,159 @@ +#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" + +/* +Future work: +Wirte comments +write in/outs for all variables clearly +*/ + +namespace scream { + +// ========================================================================================= +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::set_grids( + const std::shared_ptr grids_manager) { + + using namespace ekat::units; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); + + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + + // ------------------------------------------------------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // ------------------------------------------------------------------------------------------------------------------------- + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] + /*add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa] + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); + add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); + add_field("obklen", scalar2d_layout, m, grid_name); + add_field("surfric", scalar2d_layout, m / s, grid_name); + + auto nondim = ekat::units::Units::nondimensional(); + add_field("landfrac", scalar2d_layout, nondim, grid_name); + //add_field("icefrac", scalar2d_layout, nondim, grid_name); + //add_field("ocnfrac", scalar2d_layout, nondim, grid_name); + add_field("fv", scalar2d_layout, m / s, grid_name); + add_field("ram1", scalar2d_layout, s / m, grid_name); + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + // printf("%s \n", int_nmr_field_name); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + }*/ +} + +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + // Gather runtime options + //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); +} + +// ========================================================================================= +void MAMSrfOnlineEmiss::run_impl(const double dt) { + std::cout << "End of derydep run" << std::endl; +} + +// ========================================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp new file mode 100644 index 00000000000..1ad5e4e6da6 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -0,0 +1,62 @@ +#ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#define EAMXX_MAM_SRF_ONLINE_EMISS_HPP + +// For declaring surface and online emission class derived from atm process class +#include + +// For MAM4 aerosol configuration +#include + +// For component name +#include + +namespace scream { + +// The process responsible for handling MAM4 surface and online emissions. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + public: + // Constructor + MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_srf_online_emissions"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMSrfOnlineEmiss + +} // namespace scream + +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index aea9f2f543a..f8d3a9edb19 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -29,6 +29,7 @@ #include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" #include "physics/mam/eamxx_mam_aci_process_interface.hpp" #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" +#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -68,6 +69,7 @@ inline void register_physics () { proc_factory.register_product("mam4_drydep",&create_atmosphere_process); proc_factory.register_product("mam4_aci",&create_atmosphere_process); proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); + proc_factory.register_product("mam4_srf_online_emiss",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); From 0223b78cd7222ee45e9c9392a2c2b8fb4a01cd0b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 21 Feb 2024 17:36:04 -0800 Subject: [PATCH 02/65] Adds some inputs and preprocess struct, test works now --- ...and_online_emissions_process_interface.cpp | 96 +++++++++++++++---- ...and_online_emissions_process_interface.hpp | 60 +++++++++++- 2 files changed, 133 insertions(+), 23 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index c107dbe6970..f7396950be3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -9,7 +9,8 @@ write in/outs for all variables clearly namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -19,7 +20,6 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::Paramet // ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { - using namespace ekat::units; // The units of mixing ratio Q are technically non-dimensional. @@ -42,16 +42,17 @@ void MAMSrfOnlineEmiss::set_grids( // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - - // Layout for 2D (2d horiz) variable + const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + + // Layout for 2D (2d horiz) variable const FieldLayout scalar2d_layout{{COL}, {ncol_}}; - + // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, grid_name); // temperature [K] - /*add_field("p_mid", scalar3d_layout_mid, Pa, + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, grid_name); // total pressure @@ -67,17 +68,9 @@ void MAMSrfOnlineEmiss::set_grids( "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio - add_field("dgncur_awet", scalar4d_layout_mid, m, grid_name); - add_field("wetdens", scalar4d_layout_mid, kg / m3, grid_name); - add_field("obklen", scalar2d_layout, m, grid_name); - add_field("surfric", scalar2d_layout, m / s, grid_name); - - auto nondim = ekat::units::Units::nondimensional(); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - //add_field("icefrac", scalar2d_layout, nondim, grid_name); - //add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - add_field("fv", scalar2d_layout, m / s, grid_name); - add_field("ram1", scalar2d_layout, s / m, grid_name); + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios @@ -119,7 +112,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - }*/ + } } // ========================================================================================= @@ -140,14 +133,77 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); - EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), - "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); + + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // set up our preprocess functor + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1ad5e4e6da6..d31cd9075c6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,7 +1,8 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP -// For declaring surface and online emission class derived from atm process class +// For declaring surface and online emission class derived from atm process +// class #include // For MAM4 aerosol configuration @@ -12,12 +13,21 @@ namespace scream { -// The process responsible for handling MAM4 surface and online emissions. The AD -// stores exactly ONE instance of this class in its list of subcomponents. +// The process responsible for handling MAM4 surface and online emissions. The +// AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { + using KT = ekat::KokkosTypes; + // number of horizontal columns and vertical levels int ncol_, nlev_; + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + // buffer for sotring temporary variables mam_coupling::Buffer buffer_; @@ -54,6 +64,50 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); + team.team_barrier(); + + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMAci::Preprocess + + private: + // preprocessing scratch pad + Preprocess preprocess_; }; // MAMSrfOnlineEmiss From 30f0494b2348f24790f0bd81d19fb3af94c07edb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 22 Feb 2024 11:33:04 -0800 Subject: [PATCH 03/65] Adds some notes for the integration --- ...and_online_emissions_process_interface.cpp | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f7396950be3..f3d2a5a04ba 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -208,7 +208,50 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - std::cout << "End of derydep run" << std::endl; + + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + /* Rough notes: + + Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two + subroutines, aero_model_emissions and set_srf_emissions. + + Here is the code: + + ! initialize chemistry constituent surface fluxes to zero + do m = 2,pcnst + n = map2chm(m) + if (n>0) cam_in%cflx(:,m) = 0._r8 + enddo + + ! aerosol emissions ... + call aero_model_emissions( state, & ! in + cam_in ) ! out + + ! prescribed emissions from file ... + + !----------------------------------------------------------------------- + ! ... Set surface emissions + !----------------------------------------------------------------------- + call set_srf_emissions( lchnk, ncol, sflx(:,:) ) + + do m = 1,pcnst + n = map2chm(m) + if ( n /= h2o_ndx .and. n > 0 ) then + cam_in%cflx(:ncol,m) = cam_in%cflx(:ncol,m) + sflx(:ncol,n) + call outfld( sflxnam(m), cam_in%cflx(:ncol,m), ncol,lchnk ) + endif + enddo + + + */ + + std::cout << "End of surface emissions run" << std::endl; } // ========================================================================================= From 2bdad85e381ecc956519372e013b43b12b2c8888 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Jul 2024 06:38:07 -0700 Subject: [PATCH 04/65] A working test in new single-process test folder --- ...and_online_emissions_process_interface.cpp | 196 ++++++++++-------- .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../mam/emissions/CMakeLists.txt | 44 ++++ .../single-process/mam/emissions/input.yaml | 37 ++++ .../single-process/mam/emissions/output.yaml | 61 ++++++ 5 files changed, 253 insertions(+), 86 deletions(-) create mode 100644 components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/emissions/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/emissions/output.yaml diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f3d2a5a04ba..e41e41dc73e 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -17,20 +17,18 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - q_unit.set_string("kg/kg"); - - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); + // set grid for all the inputs and outputs + // use physics grid + grid_ = grids_manager->get_grid("Physics"); - grid_ = grids_manager->get_grid("Physics"); + // Name of the grid const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -39,93 +37,114 @@ void MAMSrfOnlineEmiss::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // Layout for 3D (2d horiz X 1d vertical) variables + // mid points + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + // layout for 2D (1d horiz X 1d vertical) variable + FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + auto nondim = ekat::units::Units::nondimensional(); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + // atmospheric quantities + // specific humidity [kg/kg] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + + // cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_layout_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_layout_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + + // planetary boundary layer height + add_field("pbl_height", scalar2d_layout_col, m, grid_name); + + // ======================================================================== + // Output from this whole process + // ======================================================================== + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - + mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - } - } - // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - // printf("%s \n", int_nmr_field_name); - - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - + mam_coupling::cld_aero_mmr_field_name(mode, a); if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } - } - } + } // end for loop num species + } // end for loop for num modes - // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } -} + } // end for loop num gases -// ========================================================================================= -// ON HOST, returns the number of bytes of device memory needed by the above -// Buffer type given the number of columns and vertical levels +} // function set_grids ends + +// ================================================================ +// INIT_BUFFERS +// ================================================================ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -// ========================================================================================= -// ON HOST, initializeŃ• the Buffer type with sufficient memory to store -// intermediate (dry) quantities on the given number of columns with the given -// number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -138,26 +157,31 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ + void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -208,7 +232,6 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -218,15 +241,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two - subroutines, aero_model_emissions and set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in + chemistry.F90. Basically call two subroutines, aero_model_emissions and + set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -235,9 +259,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -248,7 +272,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index 95e4d81ccbb..62435db229f 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -23,6 +23,7 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam/aci) add_subdirectory(mam/drydep) add_subdirectory(mam/wet_scav) + add_subdirectory(mam/emissions) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt new file mode 100644 index 00000000000..b57adcb60ef --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -0,0 +1,44 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_srf_online_emiss_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_srf_online_emiss physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_srf_online_emiss physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml new file mode 100644 index 00000000000..741cf297032 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -0,0 +1,37 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_srf_online_emiss] + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + pbl_height : 1.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml new file mode 100644 index 00000000000..7363b899a5f --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -0,0 +1,61 @@ +%YAML 1.1 +--- +filename_prefix: mam4_srf_online_emiss_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 +output_control: + Frequency: 1 + frequency_units: nsteps +... From 9948fa85ae6e7664fc2ea6fd96f08d82bd717fde Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 8 Jul 2024 11:19:45 -0700 Subject: [PATCH 05/65] Adds horiz wind vector and some cleanup; test is working --- ...and_online_emissions_process_interface.cpp | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index e41e41dc73e..fe415e9436c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -39,12 +39,14 @@ void MAMSrfOnlineEmiss::set_grids( // Layout for 3D (2d horiz X 1d vertical) variables // mid points - FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; // interfaces - FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + // vector layout + FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true,2); // layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + FieldLayout scalar2d_col{{COL}, {ncol_}}; using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers @@ -57,37 +59,40 @@ void MAMSrfOnlineEmiss::set_grids( // ------------------------------------------------------------------------------------------------------------------------- // atmospheric quantities // specific humidity [kg/kg] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud liquid mass mixing ratio [kg/kg] - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud ice mass mixing ratio [kg/kg] - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); // cloud liquid number mixing ratio [1/kg] - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); // cloud ice number mixing ratio [1/kg] - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); // Temperature[K] at midpoints - add_field("T_mid", scalar3d_layout_mid, K, grid_name); + add_field("T_mid", scalar3d_mid, K, grid_name); // Vertical pressure velocity [Pa/s] at midpoints - add_field("omega", scalar3d_layout_mid, Pa / s, grid_name); + add_field("omega", scalar3d_mid, Pa / s, grid_name); // Total pressure [Pa] at midpoints - add_field("p_mid", scalar3d_layout_mid, Pa, grid_name); + add_field("p_mid", scalar3d_mid, Pa, grid_name); // Total pressure [Pa] at interfaces - add_field("p_int", scalar3d_layout_int, Pa, grid_name); + add_field("p_int", scalar3d_int, Pa, grid_name); // Layer thickness(pdel) [Pa] at midpoints - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - // planetary boundary layer height - add_field("pbl_height", scalar2d_layout_col, m, grid_name); + // Planetary boundary layer height + add_field("pbl_height", scalar2d_col, m, grid_name); + + // Horizontal winds (U and V wind components) [m/s] + add_field("horiz_winds", vector3d_mid, m/s, grid_name); // ======================================================================== // Output from this whole process @@ -99,7 +104,7 @@ void MAMSrfOnlineEmiss::set_grids( // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); // cloudborne aerosol tracers of interest: number (n) mixing ratios @@ -107,7 +112,7 @@ void MAMSrfOnlineEmiss::set_grids( // NOT advected const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { @@ -115,7 +120,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -124,7 +129,7 @@ void MAMSrfOnlineEmiss::set_grids( const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } // end for loop num species @@ -132,7 +137,7 @@ void MAMSrfOnlineEmiss::set_grids( for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // end for loop num gases From c9f8c52fef3d0c439cabcec980b1c7a42db5121a Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 15 Apr 2024 12:23:13 -0600 Subject: [PATCH 06/65] get scream in correct place for emissions --- ...and_online_emissions_process_interface.cpp | 209 ++++++++---------- 1 file changed, 90 insertions(+), 119 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index fe415e9436c..f3d2a5a04ba 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -17,18 +17,20 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ================================================================ -// SET_GRIDS -// ================================================================ +// ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { using namespace ekat::units; - // set grid for all the inputs and outputs - // use physics grid - grid_ = grids_manager->get_grid("Physics"); + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + q_unit.set_string("kg/kg"); + + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + n_unit.set_string("#/kg"); - // Name of the grid + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank @@ -37,119 +39,93 @@ void MAMSrfOnlineEmiss::set_grids( // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variables - // mid points - FieldLayout scalar3d_mid{{COL, LEV}, {ncol_, nlev_}}; + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - FieldLayout scalar3d_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // vector layout - FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true,2); + const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; - // layout for 2D (1d horiz X 1d vertical) variable - FieldLayout scalar2d_col{{COL}, {ncol_}}; - - using namespace ekat::units; - auto q_unit = kg / kg; // units of mass mixing ratios of tracers - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - - auto nondim = ekat::units::Units::nondimensional(); + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d_layout{{COL}, {ncol_}}; // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- - // atmospheric quantities - // specific humidity [kg/kg] - add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud liquid mass mixing ratio [kg/kg] - add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud ice mass mixing ratio [kg/kg] - add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); - - // cloud liquid number mixing ratio [1/kg] - add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); - - // cloud ice number mixing ratio [1/kg] - add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); - - // Temperature[K] at midpoints - add_field("T_mid", scalar3d_mid, K, grid_name); - - // Vertical pressure velocity [Pa/s] at midpoints - add_field("omega", scalar3d_mid, Pa / s, grid_name); - - // Total pressure [Pa] at midpoints - add_field("p_mid", scalar3d_mid, Pa, grid_name); - - // Total pressure [Pa] at interfaces - add_field("p_int", scalar3d_int, Pa, grid_name); - - // Layer thickness(pdel) [Pa] at midpoints - add_field("pseudo_density", scalar3d_mid, Pa, grid_name); - - // Planetary boundary layer height - add_field("pbl_height", scalar2d_col, m, grid_name); - - // Horizontal winds (U and V wind components) [m/s] - add_field("horiz_winds", vector3d_mid, m/s, grid_name); - - // ======================================================================== - // Output from this whole process - // ======================================================================== + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // temperature [K] + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // pressure at mid points in [Pa] + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, + grid_name); // pseudo density in [Pa] + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // liquid cloud water [kg/kg] wet + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice cloud water [kg/kg] wet + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + add_field( + "omega", scalar3d_layout_mid, Pa / s, + grid_name); // Vertical pressure velocity [Pa/s] at midpoints + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios - for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { - // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = - mam_coupling::int_aero_nmr_field_name(mode); - add_field(int_nmr_field_name, scalar3d_mid, n_unit, + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - - // cloudborne aerosol tracers of interest: number (n) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected - const char *cld_nmr_field_name = - mam_coupling::cld_aero_nmr_field_name(mode); - add_field(cld_nmr_field_name, scalar3d_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(mode, a); + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_mid, q_unit, + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are - // NOT advected + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + // printf("%s \n", int_nmr_field_name); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(mode, a); + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_mid, q_unit, + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } - } // end for loop num species - } // end for loop for num modes + } + } + // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_mid, q_unit, + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); - } // end for loop num gases - -} // function set_grids ends + } +} -// ================================================================ -// INIT_BUFFERS -// ================================================================ +// ========================================================================================= +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } +// ========================================================================================= +// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG( buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), @@ -162,31 +138,26 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ================================================================ -// INITIALIZE_IMPL -// ================================================================ - +// ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); dry_atm_.T_mid = get_field_in("T_mid").get_view(); dry_atm_.p_mid = get_field_in("p_mid").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); - - // store fields converted to dry mmr from wet mmr in dry_atm_ - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -237,6 +208,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ========================================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -246,16 +218,15 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions and - set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two + subroutines, aero_model_emissions and set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -264,9 +235,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -277,7 +248,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; From 407c5f365c7d41952037684f9141e8dd054843f8 Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Mon, 8 Jul 2024 13:38:37 -0600 Subject: [PATCH 07/65] compiling and running test --- .../cime_config/namelist_defaults_scream.xml | 24 ++ ...and_online_emissions_process_interface.cpp | 354 ++++++++++++++---- ...and_online_emissions_process_interface.hpp | 15 +- .../src/physics/mam/mam_emissions_utils.hpp | 59 +++ 4 files changed, 366 insertions(+), 86 deletions(-) create mode 100644 components/eamxx/src/physics/mam/mam_emissions_utils.hpp diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index f09d10b0b32..bb36bdc2ec7 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,6 +269,30 @@ be lost if SCREAM_HACK_XML is not enabled. + + + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f3d2a5a04ba..b34186d7de8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,16 +1,23 @@ -#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" +#include +#include + +#include "share/grid/point_grid.hpp" +#include "share/io/scorpio_input.hpp" + +// for SCREAM_CIME_BUILD +#include "scream_config.h" /* Future work: -Wirte comments +Write comments write in/outs for all variables clearly */ namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, - const ekat::ParameterList ¶ms) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm& comm, + const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -24,17 +31,15 @@ void MAMSrfOnlineEmiss::set_grids( // The units of mixing ratio Q are technically non-dimensional. // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg / kg; - q_unit.set_string("kg/kg"); + Units q_unit(kg / kg, "kg/kg"); - auto n_unit = 1 / kg; // units of number mixing ratios of tracers - n_unit.set_string("#/kg"); + Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers - grid_ = grids_manager->get_grid("Physics"); - const auto &grid_name = grid_->name(); + grid_ = grids_manager->get_grid("Physics"); + const auto& grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; @@ -51,56 +56,56 @@ void MAMSrfOnlineEmiss::set_grids( // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] + grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] + grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure + grid_name); // total pressure add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] + grid_name); // pseudo density in [Pa] add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity + "tracers"); // specific humidity add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet + "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet + "tracers"); // ice cloud water [kg/kg] wet add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio + "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio + "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints + grid_name); // Vertical pressure velocity [Pa/s] at midpoints // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if (strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); // printf("%s \n", int_nmr_field_name); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if (strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } @@ -108,8 +113,8 @@ void MAMSrfOnlineEmiss::set_grids( } // aerosol-related gases: mass mixing ratios - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } @@ -123,13 +128,13 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { } // ========================================================================================= -// ON HOST, initializeŃ• the Buffer type with sufficient memory to store +// ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. -void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG( - buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { + EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= + requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); @@ -137,76 +142,264 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } +// ========================================================================================= +// inline void set_emissions_layouts( +// const std::map map_spec_id, +// const std::string emis_type, +// std::map& host_views, +// mam_coupling::complex_view_2d::HostMirror& +// specrefndxsw_host, // complex refractive index for water visible +// mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { + +// // names take the form "online_emis_specifier_for_SO2" +// for (const auto& item : map_spec_id) { +// const auto spec_name = item.first; +// const int species_id = item.second; +// const auto file_name = emis_type + "_emis_specifier_" + spec_name; +// // const auto& fname = m_params.get(file_name); +// // update file name + + +// // read data +// AtmosphereInput srf_emissions_reader( +// params_srf_emissions, grid_, host_views_emissions, layouts_emissions); +// srf_emissions_reader.read_variables(); +// srf_emissions_reader.finalize(); +// } // end ispec + + + + // for (int i = 0; i < nswbands; i++) { + // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); + // specrefndxsw_host(i, species_id).imag() = + // haero::abs(host_views[sw_im_name](i)); + // } + // for (int i = 0; i < nlwbands; i++) { + // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); + // specrefndxlw_host(i, species_id).imag() = + // haero::abs(host_views[lw_im_name](i)); + // } + +// } // end // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - wet_atm_.omega = get_field_in("omega").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + + // NOTE: these are taken as arguments to srf_emissions_inti() + // and then passed to trcdata_init() + // rmv_file = false; + // emis_cycle_yr + // emis_fixed_ymd + // emis_fixed_tod + // emis_type // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); + get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if (strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); + get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = + const char* cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if (strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); + get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = - get_field_out(gas_mmr_field_name).get_view(); + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + // // read data from file + { + // // using namespace ShortFieldTagsNames; + + using view_1d_host = typename KT::view_1d::HostMirror; + + mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + + // { + // make a list of host views + std::map host_views_srf_emissions; + // defines layouts + std::map layouts_srf_emissions; + ekat::ParameterList params_srf_emissions; + std::string prefix_srf_emissions = "srf_emis_specifier_for_"; + + // // constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; + // // auto specrefndxsw_host = mam_coupling::complex_view_2d::HostMirror( + // // "specrefndxsw_host", nswbands_, maxd_aspectype); + + // // auto specrefndxlw_host = mam_coupling::complex_view_2d::HostMirror( + // // "specrefndxlw_host", nlwbands_, maxd_aspectype); + + std::map map_srf_emiss_name_species_id; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["num_a1"] = 3; + map_srf_emiss_name_species_id["num_a2"] = 4; + map_srf_emiss_name_species_id["num_a4"] = 5; + map_srf_emiss_name_species_id["pom_a4"] = 6; + map_srf_emiss_name_species_id["so4_a1"] = 7; + map_srf_emiss_name_species_id["so4_a2"] = 8; + + std::map map_online_emiss_name_species_id; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["num_a1"] = 3; + map_online_emiss_name_species_id["num_a2"] = 4; + map_online_emiss_name_species_id["num_a4"] = 5; + map_online_emiss_name_species_id["pom_a4"] = 6; + map_online_emiss_name_species_id["so4_a1"] = 7; + map_online_emiss_name_species_id["so4_a2"] = 8; + + // To create the input object, we need to set: + // 1) names (do during read loop) + // 2) params + // 3) host views + // 4) layouts + // set_emissions_names(surname_emissions, params_srf_emissions, + // host_views_emissions, layouts_emissions); + + // inline void set_emissions_names( + // const std::map map_spec_id, + // const std::string emis_type, + // std::map& host_views, + // mam_coupling::complex_view_2d::HostMirror& + // specrefndxsw_host, // complex refractive index for water visible + // mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { + + // // names take the form "online_emis_specifier_for_SO2" + // for (const auto& item : map_spec_id) { + // const auto spec_name = item.first; + // const int species_id = item.second; + // const auto file_name = emis_type + "_emis_specifier_" + spec_name; + // const auto& fname = m_params.get(file_name); + // update file name + + + // // read data + // AtmosphereInput srf_emissions_reader( + // params_srf_emissions, grid_, host_views_emissions, layouts_emissions); + // srf_emissions_reader.read_variables(); + // srf_emissions_reader.finalize(); + // } // end ispec + + + + // for (int i = 0; i < nswbands; i++) { + // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); + // specrefndxsw_host(i, species_id).imag() = + // haero::abs(host_views[sw_im_name](i)); + // } + // for (int i = 0; i < nlwbands; i++) { + // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); + // specrefndxlw_host(i, species_id).imag() = + // haero::abs(host_views[lw_im_name](i)); + // } + + // } // end + + // =============== + // Names + // =============== + + // names take the form _emis_specifier_for_ + // set_emissions_names("srf", params_srf_emissions, host_views_srf_emissions, layouts_srf_emissions); + // set_emissions_names("online", params_srf_emissions, host_views_online_emissions, layouts_online_emissions); + + // =============== + // Params + // =============== + using strvec_t = std::vector; + params_srf_emissions.set("Skip_Grid_Checks", true); + // params_srf_emissions.set("Field Names", {refindex_real_sw, refindex_im_sw, + // refindex_real_lw, refindex_im_lw}); + // params_srf_emissions.set("Filename", fname); + + // =============== + // Host Views + // =============== + using view_1d_host = typename KT::view_1d::HostMirror; + std::map host_views; + + // host_views[refindex_real_sw] = view_1d_host(refindex_real_sw, + // nswbands); host_views[refindex_im_sw] = view_1d_host(refindex_im_sw, + // nswbands); host_views[refindex_real_lw] = + // view_1d_host(refindex_real_lw, nlwbands); host_views[refindex_im_lw] = + // view_1d_host(refindex_im_lw, nlwbands); + + // =============== + // Layouts + // =============== + std::map layouts; + // FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; + // FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; + + // layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); + // layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); + // layouts.emplace(refindex_real_lw, scalar_refindex_lw_layout); + // layouts.emplace(refindex_im_lw, scalar_refindex_lw_layout); + + // reshape specrefndxsw_host and copy it to device + // mam4::modal_aer_opt::set_device_specrefindex( + // aerosol_optics_device_data_.specrefindex_sw, "short_wave", + // specrefndxsw_host); + // mam4::modal_aer_opt::set_device_specrefindex( + // aerosol_optics_device_data_.specrefindex_lw, "long_wave", + // specrefndxlw_host); + } } -// ========================================================================================= +// ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< @@ -218,15 +411,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { /* Rough notes: - Here we should implement or port the chem_emissions subroutine in chemistry.F90. Basically call two - subroutines, aero_model_emissions and set_srf_emissions. - + Here we should implement or port the chem_emissions subroutine in + chemistry.F90. Basically call two subroutines, aero_model_emissions and + set_srf_emissions. + Here is the code: ! initialize chemistry constituent surface fluxes to zero do m = 2,pcnst n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 + if (n>0) cam_in%cflx(:,m) = 0._r8 enddo ! aerosol emissions ... @@ -235,9 +429,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { ! prescribed emissions from file ... - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- ! ... Set surface emissions - !----------------------------------------------------------------------- + !----------------------------------------------------------------------- call set_srf_emissions( lchnk, ncol, sflx(:,:) ) do m = 1,pcnst @@ -248,11 +442,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { endif enddo - + */ std::cout << "End of surface emissions run" << std::endl; } -// ========================================================================================= -} // namespace scream +// ============================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index d31cd9075c6..d59f0df56c1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,14 +1,17 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#include +#include +#include +#include +// For MAM4 aerosol configuration +#include +#include // For declaring surface and online emission class derived from atm process // class #include - -// For MAM4 aerosol configuration -#include - -// For component name +// #include #include namespace scream { @@ -113,4 +116,4 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { } // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP \ No newline at end of file +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp new file mode 100644 index 00000000000..807896f1239 --- /dev/null +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -0,0 +1,59 @@ +#ifndef MAM_EMISSIONS_READ_TABLES_HPP +#define MAM_EMISSIONS_READ_TABLES_HPP + +#include "ekat/ekat_parameter_list.hpp" +#include "mam_coupling.hpp" +#include "share/field/field_manager.hpp" +#include "share/grid/abstract_grid.hpp" +#include "share/grid/grids_manager.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +// later to mam_coupling.hpp +namespace scream::mam_coupling { + +using view_1d_host = typename KT::view_1d::HostMirror; +using view_1d_int_host = typename KT::view_1d::HostMirror; +using view_2d_host = typename KT::view_2d::HostMirror; +// using view_5d_host = typename KT::view_ND::HostMirror; +// using complex_view_1d = typename KT::view_1d>; + +// constexpr int nlwbands = mam4::modal_aer_opt::nlwbands; +// constexpr int nswbands = mam4::modal_aer_opt::nswbands; + +struct AerosolSurfaceEmissionsHostData { + // these have dim = n_species + view_1d_host emis_species_index; + view_1d_host emis_species_units; + view_1d_host emis_species_name; + // molecular weight + view_1d_host emis_species_mw; + // number of sectors in each field + view_1d_int_host emis_species_nsectors; + // FIXME: not quite sure what this does--maybe just a placeholder for fields(:, i_sector)? + view_1d_host emis_species_sector; + // note fields have dim = n_species x nsectors + // TODO: fields have units??? maybe the same as the upper spec units + view_2d_host emis_species_fields; +}; + +using AerosolSurfaceEmissionsDeviceData = + mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; + +inline void set_emissions_params( + AerosolSurfaceEmissionsHostData &aerosol_emissions_host_data, + ekat::ParameterList ¶ms_emissions, + std::map &layouts, + std::map &host_views) { + // Set up input structure to read data from file. + using strvec_t = std::vector; + using namespace ShortFieldTagsNames; + + // using SrfEmisDims = mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; + // SrfEmisDims srf_emimssions_dims; + +} + +} // namespace scream::mam_coupling + +#endif From b30d5fdf25e7416817b00cb05f30ea36f1c197cc Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 11 Jul 2024 19:01:51 -0600 Subject: [PATCH 08/65] reading files but scorpio issues with freeing file --- ...and_online_emissions_process_interface.cpp | 398 +++++++++--------- ...and_online_emissions_process_interface.hpp | 37 +- .../src/physics/mam/mam_emissions_utils.hpp | 84 ++-- .../single-process/mam/emissions/input.yaml | 26 +- 4 files changed, 301 insertions(+), 244 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index b34186d7de8..3acf7e6f5a1 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -16,8 +16,8 @@ write in/outs for all variables clearly namespace scream { // ========================================================================================= -MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm& comm, - const ekat::ParameterList& params) +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be * initialized here. Like universal constants, mam wetscav options. @@ -35,22 +35,22 @@ void MAMSrfOnlineEmiss::set_grids( Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers + // NOTE: final output with be a flux for each grid point + // e.g., flux__emissions(Nx, Ny, Nspec) + // [kg m^-2 s^-1] or [# m^-2 s^-1] grid_ = grids_manager->get_grid("Physics"); - const auto& grid_name = grid_->name(); + const auto &grid_name = grid_->name(); ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - // Define the different field layouts that will be used for this process - using namespace ShortFieldTagsNames; - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces - const FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; - const FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + const FieldLayout scalar3d_layout_mid = grid_->get_3d_scalar_layout(true); + const FieldLayout scalar3d_layout_int = grid_->get_3d_scalar_layout(false); // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout{{COL}, {ncol_}}; + const FieldLayout scalar2d_layout = grid_->get_2d_scalar_layout(); // ------------------------------------------------------------------------------------------------------------------------- // These variables are "required" or pure inputs for the process @@ -69,23 +69,24 @@ void MAMSrfOnlineEmiss::set_grids( "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice cloud water [kg/kg] wet - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, grid_name); // Vertical pressure velocity [Pa/s] at midpoints + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char* int_mmr_field_name = + const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { @@ -96,13 +97,12 @@ void MAMSrfOnlineEmiss::set_grids( } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - // printf("%s \n", int_nmr_field_name); + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char* cld_mmr_field_name = + const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if (strlen(cld_mmr_field_name) > 0) { @@ -114,7 +114,7 @@ void MAMSrfOnlineEmiss::set_grids( // aerosol-related gases: mass mixing ratios for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } @@ -131,7 +131,7 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { // ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. -void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), "Error! Insufficient buffer size.\n"); @@ -143,7 +143,42 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } // ========================================================================================= +// // TODO: comments! +// void MAMSrfOnlineEmiss::set_emissions_names( +// const std::map map_spec_id, const std::string +// emis_type, const ekat::ParameterList &m_params, std::map &host_views) { + +// using view_1d_host = typename KT::view_1d::HostMirror; + +// // names take the form online_emis_specifier_for_ +// for (const auto &item : map_spec_id) { +// const auto spec_name = item.first; +// const int species_id = item.second; +// const auto file_name = emis_type + "_emis_specifier_" + spec_name; +// const auto &fname = m_params.get(file_name); + +// // read data +// AtmosphereInput srf_emissions_reader(m_params, grid_, +// host_views_emissions, +// layouts_emissions); +// srf_emissions_reader.read_variables(); +// srf_emissions_reader.finalize(); +// } // end ispec +// // for (int i = 0; i < nswbands; i++) { +// // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); +// // specrefndxsw_host(i, species_id).imag() = +// // haero::abs(host_views[sw_im_name](i)); +// // } +// // for (int i = 0; i < nlwbands; i++) { +// // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); +// // specrefndxlw_host(i, species_id).imag() = +// // haero::abs(host_views[lw_im_name](i)); +// // } + +// } // end set_emissions_names +// ========================================================================================= // inline void set_emissions_layouts( // const std::map map_spec_id, // const std::string emis_type, @@ -160,26 +195,24 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager& buffer_manager) { // // const auto& fname = m_params.get(file_name); // // update file name - // // read data // AtmosphereInput srf_emissions_reader( -// params_srf_emissions, grid_, host_views_emissions, layouts_emissions); +// params_srf_emissions, grid_, host_views_emissions, +// layouts_emissions); // srf_emissions_reader.read_variables(); // srf_emissions_reader.finalize(); // } // end ispec - - - // for (int i = 0; i < nswbands; i++) { - // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); - // specrefndxsw_host(i, species_id).imag() = - // haero::abs(host_views[sw_im_name](i)); - // } - // for (int i = 0; i < nlwbands; i++) { - // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); - // specrefndxlw_host(i, species_id).imag() = - // haero::abs(host_views[lw_im_name](i)); - // } +// for (int i = 0; i < nswbands; i++) { +// specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); +// specrefndxsw_host(i, species_id).imag() = +// haero::abs(host_views[sw_im_name](i)); +// } +// for (int i = 0; i < nlwbands; i++) { +// specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); +// specrefndxlw_host(i, species_id).imag() = +// haero::abs(host_views[lw_im_name](i)); +// } // } // end // ========================================================================================= @@ -187,16 +220,17 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); dry_atm_.qv = buffer_.qv_dry; dry_atm_.qc = buffer_.qc_dry; dry_atm_.nc = buffer_.nc_dry; @@ -215,40 +249,41 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // number (n) mixing ratios for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); + get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char* cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); + get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char* int_mmr_field_name = + const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); if (strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); + get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char* cld_mmr_field_name = + const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); if (strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); + get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } @@ -256,148 +291,125 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); - // // read data from file - { - // // using namespace ShortFieldTagsNames; - - using view_1d_host = typename KT::view_1d::HostMirror; - - mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; - - // { - // make a list of host views - std::map host_views_srf_emissions; - // defines layouts - std::map layouts_srf_emissions; - ekat::ParameterList params_srf_emissions; - std::string prefix_srf_emissions = "srf_emis_specifier_for_"; - - // // constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; - // // auto specrefndxsw_host = mam_coupling::complex_view_2d::HostMirror( - // // "specrefndxsw_host", nswbands_, maxd_aspectype); - - // // auto specrefndxlw_host = mam_coupling::complex_view_2d::HostMirror( - // // "specrefndxlw_host", nlwbands_, maxd_aspectype); - - std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; - map_srf_emiss_name_species_id["num_a1"] = 3; - map_srf_emiss_name_species_id["num_a2"] = 4; - map_srf_emiss_name_species_id["num_a4"] = 5; - map_srf_emiss_name_species_id["pom_a4"] = 6; - map_srf_emiss_name_species_id["so4_a1"] = 7; - map_srf_emiss_name_species_id["so4_a2"] = 8; - - std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; - map_online_emiss_name_species_id["num_a1"] = 3; - map_online_emiss_name_species_id["num_a2"] = 4; - map_online_emiss_name_species_id["num_a4"] = 5; - map_online_emiss_name_species_id["pom_a4"] = 6; - map_online_emiss_name_species_id["so4_a1"] = 7; - map_online_emiss_name_species_id["so4_a2"] = 8; - - // To create the input object, we need to set: - // 1) names (do during read loop) - // 2) params - // 3) host views - // 4) layouts - // set_emissions_names(surname_emissions, params_srf_emissions, - // host_views_emissions, layouts_emissions); - - // inline void set_emissions_names( - // const std::map map_spec_id, - // const std::string emis_type, - // std::map& host_views, - // mam_coupling::complex_view_2d::HostMirror& - // specrefndxsw_host, // complex refractive index for water visible - // mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { - - // // names take the form "online_emis_specifier_for_SO2" - // for (const auto& item : map_spec_id) { - // const auto spec_name = item.first; - // const int species_id = item.second; - // const auto file_name = emis_type + "_emis_specifier_" + spec_name; - // const auto& fname = m_params.get(file_name); - // update file name - - - // // read data - // AtmosphereInput srf_emissions_reader( - // params_srf_emissions, grid_, host_views_emissions, layouts_emissions); - // srf_emissions_reader.read_variables(); - // srf_emissions_reader.finalize(); - // } // end ispec - - - - // for (int i = 0; i < nswbands; i++) { - // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); - // specrefndxsw_host(i, species_id).imag() = - // haero::abs(host_views[sw_im_name](i)); - // } - // for (int i = 0; i < nlwbands; i++) { - // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); - // specrefndxlw_host(i, species_id).imag() = - // haero::abs(host_views[lw_im_name](i)); - // } - - // } // end - - // =============== - // Names - // =============== - - // names take the form _emis_specifier_for_ - // set_emissions_names("srf", params_srf_emissions, host_views_srf_emissions, layouts_srf_emissions); - // set_emissions_names("online", params_srf_emissions, host_views_online_emissions, layouts_online_emissions); - - // =============== - // Params - // =============== - using strvec_t = std::vector; - params_srf_emissions.set("Skip_Grid_Checks", true); - // params_srf_emissions.set("Field Names", {refindex_real_sw, refindex_im_sw, - // refindex_real_lw, refindex_im_lw}); - // params_srf_emissions.set("Filename", fname); - - // =============== - // Host Views - // =============== - using view_1d_host = typename KT::view_1d::HostMirror; - std::map host_views; - - // host_views[refindex_real_sw] = view_1d_host(refindex_real_sw, - // nswbands); host_views[refindex_im_sw] = view_1d_host(refindex_im_sw, - // nswbands); host_views[refindex_real_lw] = - // view_1d_host(refindex_real_lw, nlwbands); host_views[refindex_im_lw] = - // view_1d_host(refindex_im_lw, nlwbands); - - // =============== - // Layouts - // =============== - std::map layouts; - // FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; - // FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; - - // layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); - // layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); - // layouts.emplace(refindex_real_lw, scalar_refindex_lw_layout); - // layouts.emplace(refindex_im_lw, scalar_refindex_lw_layout); - - // reshape specrefndxsw_host and copy it to device - // mam4::modal_aer_opt::set_device_specrefindex( - // aerosol_optics_device_data_.specrefindex_sw, "short_wave", - // specrefndxsw_host); - // mam4::modal_aer_opt::set_device_specrefindex( - // aerosol_optics_device_data_.specrefindex_lw, "long_wave", - // specrefndxlw_host); - } -} + // read data from files + using view_1d_host = typename KT::view_1d::HostMirror; + using view_2d_host = typename KT::view_2d::HostMirror; + using strvec_t = std::vector; + + // mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + + std::map map_srf_emiss_name_species_id; + strvec_t srf_emiss_spec_names; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["num_a1"] = 3; + map_srf_emiss_name_species_id["num_a2"] = 4; + map_srf_emiss_name_species_id["num_a4"] = 5; + map_srf_emiss_name_species_id["pom_a4"] = 6; + map_srf_emiss_name_species_id["so4_a1"] = 7; + map_srf_emiss_name_species_id["so4_a2"] = 8; + // for (const auto &item : map_srf_emiss_name_species_id) { + // srf_emiss_spec_names.push_back(item.first); + // } + + std::map map_online_emiss_name_species_id; + strvec_t online_emiss_spec_names; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["num_a1"] = 3; + map_online_emiss_name_species_id["num_a2"] = 4; + map_online_emiss_name_species_id["num_a4"] = 5; + map_online_emiss_name_species_id["pom_a4"] = 6; + map_online_emiss_name_species_id["so4_a1"] = 7; + map_online_emiss_name_species_id["so4_a2"] = 8; + // for (const auto &item : map_srf_emiss_name_species_id) { + // online_emiss_spec_names.push_back(item.first); + // } + + // To create the input object, we need to: + // - set names of views + // - declare host views + // - initialize FieldLayouts + // - initialize params + + using namespace ShortFieldTagsNames; + + // make a list of host views, that holds nspec-dimensional views of + // srf/online emissions at a grid point + std::map host_views_srf_emiss; + // list of layouts of srf/online emissions views + // NOTE: these are the same, but it seems best (necessary?) to have + // equally-sized lists + std::map layouts_srf_emiss; + ekat::ParameterList params_srf_emiss; + std::string middle_name_emiss = "_emis_specifier_for_"; + + // TODO: break this out into a function in emissions_utils.hpp + // host_views_srf_emiss[sname] = view_1d_host(sname, mam_coupling::n_srf_emiss); + // layouts_srf_emiss.emplace(sname, scalar_srf_emiss_layout); + // for (const auto &item : map_srf_emiss_name_species_id) { + // online_emiss_spec_names.push_back(item.first); + // const auto oname = item.first; + // host_views_online_emiss[oname] = view_2d_host(oname, ) + // } + + // set names of views + const std::string srf_emiss_name = "surface_emissions"; + // const std::string online_emiss_name = "online_emissions"; + + // declare the host views + host_views_srf_emiss[srf_emiss_name] = view_1d_host(srf_emiss_name, 1); + // host_views_emissions[online_emiss_name] = + // view_1d_host(online_emiss_name, mam_coupling::n_online_emiss); + + // initialize and collect FieldLayouts + // layout for 2D (2d horiz == 1d flattened col index) scalar-valued variable + FieldLayout scalar_srf_emiss_layout{{COL}, {ncol_}, {srf_emiss_name}}; + // FieldLayout scalar_online_emiss_layout{ + // {CMP}, {mam_coupling::n_online_emiss}, {online_emiss_name}}; + layouts_srf_emiss.emplace(srf_emiss_name, scalar_srf_emiss_layout); + // layouts_emissions.emplace(online_emiss_name, scalar_online_emiss_layout); + + // initialize params + params_srf_emiss.set("Skip_Grid_Checks", true); + // these need to be the emission-type names from the map__emiss_name_species_id maps + // params_srf_emiss.set("Field Names", {srf_emiss_name}); + + // namelist entries take the form _emis_specifier_for_ + std::string emis_type = "srf"; + for (const auto &item : map_srf_emiss_name_species_id) { + const auto spec_name = item.first; + const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + spec_name; + const auto &fpath = m_params.get(file_name); + params_srf_emiss.set("Filename", fpath); + + // read data + AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); + srf_emissions_reader.read_variables(); + srf_emissions_reader.finalize(); + // copy data to device + + } // end ispec + emis_type = "online"; + // for (const auto &item : map_online_emiss_name_species_id) { + // const auto spec_name = item.first; + // const int species_id = item.second; + // const auto file_name = emis_type + middle_name_emiss + spec_name; + // const auto &fname = m_params.get(file_name); + // params_emissions.set("Filename", fname); + + // // read data + // AtmosphereInput online_emissions_reader( + // m_params, grid_, host_views_emissions, layouts_emissions); + // online_emissions_reader.read_variables(); + // online_emissions_reader.finalize(); + // // copy data to device + + // } // end ispec +} // end initialize_impl() // ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index d59f0df56c1..0abd1a50e85 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -14,6 +14,15 @@ // #include #include +// TODO: determine when these may be necessary +// #ifndef KOKKOS_ENABLE_CUDA +// #define protected_except_cuda public +// #define private_except_cuda public +// #else +// #define protected_except_cuda protected +// #define private_except_cuda private +// #endif + namespace scream { // The process responsible for handling MAM4 surface and online emissions. The @@ -37,7 +46,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - public: +public: // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -52,8 +61,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::string name() const { return "mam_srf_online_emissions"; } // grid - void set_grids( - const std::shared_ptr grids_manager) override; + void + set_grids(const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -78,25 +87,25 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { const mam_coupling::AerosolState &wet_aero, const mam_coupling::DryAtmosphere &dry_atm, const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; + dry_atm_pre_ = dry_atm; dry_aero_pre_ = dry_aero; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index + const int i = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); - } // operator() + } // operator() // local variables for preprocess struct // number of horizontal columns and vertical levels @@ -106,14 +115,14 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMAci::Preprocess - private: +private: // preprocessing scratch pad Preprocess preprocess_; -}; // MAMSrfOnlineEmiss +}; // MAMSrfOnlineEmiss -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp index 807896f1239..8549b47dd8c 100644 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -18,41 +18,55 @@ using view_2d_host = typename KT::view_2d::HostMirror; // using view_5d_host = typename KT::view_ND::HostMirror; // using complex_view_1d = typename KT::view_1d>; -// constexpr int nlwbands = mam4::modal_aer_opt::nlwbands; -// constexpr int nswbands = mam4::modal_aer_opt::nswbands; - -struct AerosolSurfaceEmissionsHostData { - // these have dim = n_species - view_1d_host emis_species_index; - view_1d_host emis_species_units; - view_1d_host emis_species_name; - // molecular weight - view_1d_host emis_species_mw; - // number of sectors in each field - view_1d_int_host emis_species_nsectors; - // FIXME: not quite sure what this does--maybe just a placeholder for fields(:, i_sector)? - view_1d_host emis_species_sector; - // note fields have dim = n_species x nsectors - // TODO: fields have units??? maybe the same as the upper spec units - view_2d_host emis_species_fields; -}; - -using AerosolSurfaceEmissionsDeviceData = - mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; - -inline void set_emissions_params( - AerosolSurfaceEmissionsHostData &aerosol_emissions_host_data, - ekat::ParameterList ¶ms_emissions, - std::map &layouts, - std::map &host_views) { - // Set up input structure to read data from file. - using strvec_t = std::vector; - using namespace ShortFieldTagsNames; - - // using SrfEmisDims = mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; - // SrfEmisDims srf_emimssions_dims; - -} +constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; +constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; + +using namespace ShortFieldTagsNames; + +// struct AerosolSurfaceEmissionsHostData { +// // these have dim = n_species +// view_1d_host emis_species_index; +// view_1d_host emis_species_units; +// view_1d_host emis_species_name; +// // molecular weight +// view_1d_host emis_species_mw; +// // number of sectors in each field +// view_1d_int_host emis_species_nsectors; +// // FIXME: not quite sure what this does--maybe just a placeholder for +// // fields(:, i_sector)? +// view_1d_host emis_species_sector; +// // note fields have dim = n_species x nsectors +// // TODO: fields have units??? maybe the same as the upper spec units +// view_2d_host emis_species_fields; +// }; + +// using AerosolSurfaceEmissionsDeviceData = + // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; + +// inline void set_emissions_params( +// AerosolSurfaceEmissionsHostData& aerosol_emissions_host_data, +// ekat::ParameterList& params_emissions, +// std::map& layouts, +// std::map& host_views) { +// // Set up input structure to read data from file. +// using strvec_t = std::vector; +// // using namespace ShortFieldTagsNames; + +// // using SrfEmisDims = +// // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; SrfEmisDims +// // srf_emimssions_dims; +// } + +// inline void set_emissions_names(const std::map map_spec_id, +// const std::string emis_type, +// const ekat::ParameterList& m_params, +// std::map& host_views) { + +// using view_1d_host = typename KT::view_1d::HostMirror; + +// std::string + +// } // end set_emissions_names } // namespace scream::mam_coupling diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 741cf297032..30861b656e6 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -10,7 +10,29 @@ time_stepping: atmosphere_processes: atm_procs_list: [mam4_srf_online_emiss] - + mam4_srf_online_emiss: + # MAM4xx-Surface-Emissions + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + # MAM4xx-Online-Emissions + online_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_SOAG: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc + online_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + + grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -30,7 +52,7 @@ initial_conditions: #we should get the following variables from other processes pbl_height : 1.0 - + # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From 79b88b7862c3ec769dfcea5151c88a08194fffea Mon Sep 17 00:00:00 2001 From: Michael J Schmidt Date: Thu, 18 Jul 2024 17:36:54 -0600 Subject: [PATCH 09/65] surface and online emissions data files successfull read in --- ...and_online_emissions_process_interface.cpp | 236 +++++++----------- .../src/physics/mam/mam_emissions_utils.hpp | 18 ++ .../single-process/mam/optics/CMakeLists.txt | 4 +- 3 files changed, 113 insertions(+), 145 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 3acf7e6f5a1..72171a03541 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -144,77 +144,11 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { } // ========================================================================================= // // TODO: comments! -// void MAMSrfOnlineEmiss::set_emissions_names( -// const std::map map_spec_id, const std::string -// emis_type, const ekat::ParameterList &m_params, std::map &host_views) { - -// using view_1d_host = typename KT::view_1d::HostMirror; - -// // names take the form online_emis_specifier_for_ -// for (const auto &item : map_spec_id) { -// const auto spec_name = item.first; -// const int species_id = item.second; -// const auto file_name = emis_type + "_emis_specifier_" + spec_name; -// const auto &fname = m_params.get(file_name); - -// // read data -// AtmosphereInput srf_emissions_reader(m_params, grid_, -// host_views_emissions, -// layouts_emissions); -// srf_emissions_reader.read_variables(); -// srf_emissions_reader.finalize(); -// } // end ispec - -// // for (int i = 0; i < nswbands; i++) { -// // specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); -// // specrefndxsw_host(i, species_id).imag() = -// // haero::abs(host_views[sw_im_name](i)); -// // } -// // for (int i = 0; i < nlwbands; i++) { -// // specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); -// // specrefndxlw_host(i, species_id).imag() = -// // haero::abs(host_views[lw_im_name](i)); -// // } - -// } // end set_emissions_names +// void MAMSrfOnlineEmiss::set_emissions_names() {} \\ end set_emissions_names() // ========================================================================================= -// inline void set_emissions_layouts( -// const std::map map_spec_id, -// const std::string emis_type, -// std::map& host_views, -// mam_coupling::complex_view_2d::HostMirror& -// specrefndxsw_host, // complex refractive index for water visible -// mam_coupling::complex_view_2d::HostMirror& specrefndxlw_host) { - -// // names take the form "online_emis_specifier_for_SO2" -// for (const auto& item : map_spec_id) { -// const auto spec_name = item.first; -// const int species_id = item.second; -// const auto file_name = emis_type + "_emis_specifier_" + spec_name; -// // const auto& fname = m_params.get(file_name); -// // update file name - -// // read data -// AtmosphereInput srf_emissions_reader( -// params_srf_emissions, grid_, host_views_emissions, -// layouts_emissions); -// srf_emissions_reader.read_variables(); -// srf_emissions_reader.finalize(); -// } // end ispec - -// for (int i = 0; i < nswbands; i++) { -// specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); -// specrefndxsw_host(i, species_id).imag() = -// haero::abs(host_views[sw_im_name](i)); -// } -// for (int i = 0; i < nlwbands; i++) { -// specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); -// specrefndxlw_host(i, species_id).imag() = -// haero::abs(host_views[lw_im_name](i)); -// } - -// } // end +// inline void set_emissions_layouts() { + +// } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Gather runtime options @@ -294,12 +228,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // read data from files using view_1d_host = typename KT::view_1d::HostMirror; using view_2d_host = typename KT::view_2d::HostMirror; - using strvec_t = std::vector; - - // mam_coupling::AerosolSurfaceEmissionsHostData srf_emissions_host_data; + // these probably belong in mam4xx std::map map_srf_emiss_name_species_id; - strvec_t srf_emiss_spec_names; map_srf_emiss_name_species_id["DMS"] = 0; map_srf_emiss_name_species_id["SO2"] = 1; map_srf_emiss_name_species_id["bc_a4"] = 2; @@ -309,12 +240,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { map_srf_emiss_name_species_id["pom_a4"] = 6; map_srf_emiss_name_species_id["so4_a1"] = 7; map_srf_emiss_name_species_id["so4_a2"] = 8; - // for (const auto &item : map_srf_emiss_name_species_id) { - // srf_emiss_spec_names.push_back(item.first); - // } + // these probably belong in mam4xx std::map map_online_emiss_name_species_id; - strvec_t online_emiss_spec_names; map_online_emiss_name_species_id["SO2"] = 0; map_online_emiss_name_species_id["SOAG"] = 1; map_online_emiss_name_species_id["bc_a4"] = 2; @@ -324,65 +252,52 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { map_online_emiss_name_species_id["pom_a4"] = 6; map_online_emiss_name_species_id["so4_a1"] = 7; map_online_emiss_name_species_id["so4_a2"] = 8; - // for (const auto &item : map_srf_emiss_name_species_id) { - // online_emiss_spec_names.push_back(item.first); - // } - - // To create the input object, we need to: - // - set names of views - // - declare host views - // - initialize FieldLayouts - // - initialize params using namespace ShortFieldTagsNames; - // make a list of host views, that holds nspec-dimensional views of - // srf/online emissions at a grid point - std::map host_views_srf_emiss; - // list of layouts of srf/online emissions views - // NOTE: these are the same, but it seems best (necessary?) to have - // equally-sized lists - std::map layouts_srf_emiss; ekat::ParameterList params_srf_emiss; + ekat::ParameterList params_online_emiss; std::string middle_name_emiss = "_emis_specifier_for_"; - // TODO: break this out into a function in emissions_utils.hpp - // host_views_srf_emiss[sname] = view_1d_host(sname, mam_coupling::n_srf_emiss); - // layouts_srf_emiss.emplace(sname, scalar_srf_emiss_layout); - // for (const auto &item : map_srf_emiss_name_species_id) { - // online_emiss_spec_names.push_back(item.first); - // const auto oname = item.first; - // host_views_online_emiss[oname] = view_2d_host(oname, ) - // } - - // set names of views - const std::string srf_emiss_name = "surface_emissions"; - // const std::string online_emiss_name = "online_emissions"; - - // declare the host views - host_views_srf_emiss[srf_emiss_name] = view_1d_host(srf_emiss_name, 1); - // host_views_emissions[online_emiss_name] = - // view_1d_host(online_emiss_name, mam_coupling::n_online_emiss); - - // initialize and collect FieldLayouts - // layout for 2D (2d horiz == 1d flattened col index) scalar-valued variable - FieldLayout scalar_srf_emiss_layout{{COL}, {ncol_}, {srf_emiss_name}}; - // FieldLayout scalar_online_emiss_layout{ - // {CMP}, {mam_coupling::n_online_emiss}, {online_emiss_name}}; - layouts_srf_emiss.emplace(srf_emiss_name, scalar_srf_emiss_layout); - // layouts_emissions.emplace(online_emiss_name, scalar_online_emiss_layout); - // initialize params params_srf_emiss.set("Skip_Grid_Checks", true); - // these need to be the emission-type names from the map__emiss_name_species_id maps - // params_srf_emiss.set("Field Names", {srf_emiss_name}); + params_online_emiss.set("Skip_Grid_Checks", true); + + // FIXME: this, and all places used, will need to change to a flattened column index + int numlat_srf = scream::mam_coupling::nlat_srf; + int numlon_srf = scream::mam_coupling::nlon_srf; + int numcol_fake_srf = numlat_srf * numlon_srf; + const auto srf_emiss_var_names = mam4::mo_srf_emissions::srf_emimssions_data_fields; + view_2d_host srf_emiss_host; + + // TODO: break this out into a function in emissions_utils.hpp // namelist entries take the form _emis_specifier_for_ std::string emis_type = "srf"; for (const auto &item : map_srf_emiss_name_species_id) { - const auto spec_name = item.first; - const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + spec_name; + const std::string srf_emiss_name = item.first; + // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 + // Is this correct or an error in the data? + if (srf_emiss_name == "DMS") { + numlat_srf = 180; + numlon_srf = 360; + numcol_fake_srf = numlat_srf * numlon_srf; + } else { + numlat_srf = scream::mam_coupling::nlat_srf; + numlon_srf = scream::mam_coupling::nlon_srf; + numcol_fake_srf = numlat_srf * numlon_srf; + } + std::map host_views_srf_emiss; + std::map layouts_srf_emiss; + // FIXME: this will need to change to a flattened column index + FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, {"lat", "lon"}); + for ( const auto &var_name : srf_emiss_var_names.at(srf_emiss_name) ) { + host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); + layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); + } + // const auto spec_name = item.first; + // const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; const auto &fpath = m_params.get(file_name); params_srf_emiss.set("Filename", fpath); @@ -390,25 +305,60 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); srf_emissions_reader.read_variables(); srf_emissions_reader.finalize(); - // copy data to device - - } // end ispec + // copy data into host view for sending, columnwise, to mam4xx's mo_srf_emissions + // TODO: any reason not to just copy to device here? + // TODO: Could probably be a parfor here + // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { + // srf_emiss_host(colidx_fake, species_id) = host_views_srf_emiss[srf_emiss_name](colidx_fake); + // } + } // end item + + // FIXME: this, and all places used, will need to change to a flattened column index + const int numlat_online = scream::mam_coupling::nlat_online; + const int numlon_online = scream::mam_coupling::nlon_online; + int numalti_online = scream::mam_coupling::nalti_online; + int numcol_fake_online = numlat_online * numlon_online * numalti_online; + const auto online_emiss_var_names = mam4::aero_model_emissions::online_emimssions_data_fields; + + // view_3d_host online_emiss_host; emis_type = "online"; - // for (const auto &item : map_online_emiss_name_species_id) { - // const auto spec_name = item.first; - // const int species_id = item.second; - // const auto file_name = emis_type + middle_name_emiss + spec_name; - // const auto &fname = m_params.get(file_name); - // params_emissions.set("Filename", fname); - - // // read data - // AtmosphereInput online_emissions_reader( - // m_params, grid_, host_views_emissions, layouts_emissions); - // online_emissions_reader.read_variables(); - // online_emissions_reader.finalize(); - // // copy data to device - - // } // end ispec + for (const auto &item : map_online_emiss_name_species_id) { + const std::string online_emiss_name = item.first; + // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 + // Is this correct or an error in the data? + if (online_emiss_name == "SOAG") { + numalti_online = 12; + numcol_fake_online = numlat_online * numlon_online * numalti_online; + } else { + numalti_online = 13; + numcol_fake_online = numlat_online * numlon_online * numalti_online; + } + std::map host_views_online_emiss; + std::map layouts_online_emiss; + // FIXME: this will need to change to a flattened column index + FieldLayout scalar_online_emiss_layout({CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, {"altitude", "lat", "lon"}); + for ( const auto &var_name : online_emiss_var_names.at(online_emiss_name) ) { + host_views_online_emiss[var_name] = view_1d_host(var_name, numcol_fake_online); + layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); + } // end var_name + // const auto spec_name = item.first; + // const int species_id = item.second; + const auto file_name = emis_type + middle_name_emiss + online_emiss_name; + const auto &fpath = m_params.get(file_name); + params_online_emiss.set("Filename", fpath); + + // read data + AtmosphereInput online_emissions_reader(params_online_emiss, grid_, host_views_online_emiss, layouts_online_emiss); + online_emissions_reader.read_variables(); + online_emissions_reader.finalize(); + // copy data into host view for sending, columnwise, to mam4xx's mo_online_emissions + // TODO: any reason not to just copy to device here? + // TODO: Could probably be a parfor here + // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { + // online_emiss_host(colidx_fake_online, species_id) = host_views_online_emiss[online_emiss_name](colidx_fake); + // } + } // end item + } // end initialize_impl() // ============================================================================= diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp index 8549b47dd8c..527efd5de18 100644 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp @@ -21,8 +21,26 @@ using view_2d_host = typename KT::view_2d::HostMirror; constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; +// FIXME: this will need to change when we remap to flattened column idx +constexpr int nlat_srf = 96; +constexpr int nlon_srf = 144; + +constexpr int nalti_online = 13; +constexpr int nlat_online = 96; +constexpr int nlon_online = 144; + using namespace ShortFieldTagsNames; +// std::map> map_srf_emiss_file_vars; + +inline void set_file_var_names(std::map> &var_map, + std::map &spec_map) { + // for (const auto &spec : spec_map) { + // std::string spec_name = spec.first; + // std::cout << "var_map[spec_name] = " << var_map[spec_name] << "\n"; + // } +} + // struct AerosolSurfaceEmissionsHostData { // // these have dim = n_species // view_1d_host emis_species_index; diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt index a3e6f64ec7d..50f2cfa68a3 100644 --- a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -40,7 +40,7 @@ set (TEST_INPUT_FILES scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc scream/mam4xx/physprops/poly_rrtmg_c20240206.nc - scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + # scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) @@ -63,4 +63,4 @@ if (SCREAM_ENABLE_BASELINE_TESTS) # Note: one is enough, since we already check that np1 is BFB with npX set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) -endif() \ No newline at end of file +endif() From f5da2455281e8cd2d19c3bb146d7aa367c45e761 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 07:21:43 -0700 Subject: [PATCH 10/65] Horizontal remapper create_horiz_remapper works with so2 srf file --- .../mam4xx/srf_online_emiss/shell_commands | 13 ++ ...and_online_emissions_process_interface.cpp | 176 ++++++++++-------- ...and_online_emissions_process_interface.hpp | 37 ++-- .../eamxx/src/physics/mam/srf_emission.hpp | 22 +++ .../src/physics/mam/srf_emission_impl.hpp | 86 +++++++++ 5 files changed, 244 insertions(+), 90 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands create mode 100644 components/eamxx/src/physics/mam/srf_emission.hpp create mode 100644 components/eamxx/src/physics/mam/srf_emission_impl.hpp diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands new file mode 100644 index 00000000000..6995cd8b856 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands @@ -0,0 +1,13 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +#modify initial condition file to get aerosol species ICs +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b + +# Add spa as RRTMG needs spa +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 72171a03541..a694e63a5e0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -33,16 +33,16 @@ void MAMSrfOnlineEmiss::set_grids( // Nevertheless, for output reasons, we like to see 'kg/kg'. Units q_unit(kg / kg, "kg/kg"); - Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers + Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers // NOTE: final output with be a flux for each grid point // e.g., flux__emissions(Nx, Ny, Nspec) // [kg m^-2 s^-1] or [# m^-2 s^-1] - grid_ = grids_manager->get_grid("Physics"); + grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and // interfaces @@ -56,56 +56,56 @@ void MAMSrfOnlineEmiss::set_grids( // These variables are "required" or pure inputs for the process // ------------------------------------------------------------------------------------------------------------------------- add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] + grid_name); // temperature [K] add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] + grid_name); // pressure at mid points in [Pa] add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure + grid_name); // total pressure add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] + grid_name); // pseudo density in [Pa] add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity + "tracers"); // specific humidity add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet + "tracers"); // liquid cloud water [kg/kg] wet add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet + "tracers"); // ice cloud water [kg/kg] wet add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio + "tracers"); // ice number mixing ratio add_field( "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints + grid_name); // Vertical pressure velocity [Pa/s] at midpoints add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio + "tracers"); // cloud liquid wet number mixing ratio // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing // ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { + if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } } } // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name); - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { + if(strlen(cld_mmr_field_name) > 0) { add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name); } @@ -113,11 +113,21 @@ void MAMSrfOnlineEmiss::set_grids( } // aerosol-related gases: mass mixing ratios - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); } + + // Reading so2 srf emiss data + + std::string so2_data_file = + "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" + "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; + std::string srf_map_file = ""; + // Init horizontal remap + srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( + grid_, so2_data_file, srf_map_file, m_iop != nullptr); } // ========================================================================================= @@ -132,9 +142,9 @@ size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= - requested_buffer_size_in_bytes(), - "Error! Insufficient buffer size.\n"); + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); @@ -181,7 +191,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios - for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = @@ -194,11 +204,11 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { get_field_out(cld_nmr_field_name).get_view(); dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if (strlen(int_mmr_field_name) > 0) { + if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; @@ -207,14 +217,14 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios const char *cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if (strlen(cld_mmr_field_name) > 0) { + if(strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } - for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); @@ -231,9 +241,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // these probably belong in mam4xx std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; + map_srf_emiss_name_species_id["DMS"] = 0; + map_srf_emiss_name_species_id["SO2"] = 1; + map_srf_emiss_name_species_id["bc_a4"] = 2; map_srf_emiss_name_species_id["num_a1"] = 3; map_srf_emiss_name_species_id["num_a2"] = 4; map_srf_emiss_name_species_id["num_a4"] = 5; @@ -243,9 +253,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // these probably belong in mam4xx std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; + map_online_emiss_name_species_id["SO2"] = 0; + map_online_emiss_name_species_id["SOAG"] = 1; + map_online_emiss_name_species_id["bc_a4"] = 2; map_online_emiss_name_species_id["num_a1"] = 3; map_online_emiss_name_species_id["num_a2"] = 4; map_online_emiss_name_species_id["num_a4"] = 5; @@ -263,107 +273,121 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { params_srf_emiss.set("Skip_Grid_Checks", true); params_online_emiss.set("Skip_Grid_Checks", true); - // FIXME: this, and all places used, will need to change to a flattened column index - int numlat_srf = scream::mam_coupling::nlat_srf; - int numlon_srf = scream::mam_coupling::nlon_srf; + // FIXME: this, and all places used, will need to change to a flattened column + // index + int numlat_srf = scream::mam_coupling::nlat_srf; + int numlon_srf = scream::mam_coupling::nlon_srf; int numcol_fake_srf = numlat_srf * numlon_srf; - const auto srf_emiss_var_names = mam4::mo_srf_emissions::srf_emimssions_data_fields; + const auto srf_emiss_var_names = + mam4::mo_srf_emissions::srf_emimssions_data_fields; view_2d_host srf_emiss_host; // TODO: break this out into a function in emissions_utils.hpp // namelist entries take the form _emis_specifier_for_ std::string emis_type = "srf"; - for (const auto &item : map_srf_emiss_name_species_id) { + for(const auto &item : map_srf_emiss_name_species_id) { const std::string srf_emiss_name = item.first; // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 // Is this correct or an error in the data? - if (srf_emiss_name == "DMS") { - numlat_srf = 180; - numlon_srf = 360; + if(srf_emiss_name == "DMS") { + numlat_srf = 180; + numlon_srf = 360; numcol_fake_srf = numlat_srf * numlon_srf; } else { - numlat_srf = scream::mam_coupling::nlat_srf; - numlon_srf = scream::mam_coupling::nlon_srf; + numlat_srf = scream::mam_coupling::nlat_srf; + numlon_srf = scream::mam_coupling::nlon_srf; numcol_fake_srf = numlat_srf * numlon_srf; } std::map host_views_srf_emiss; std::map layouts_srf_emiss; // FIXME: this will need to change to a flattened column index - FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, {"lat", "lon"}); - for ( const auto &var_name : srf_emiss_var_names.at(srf_emiss_name) ) { + FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, + {"lat", "lon"}); + for(const auto &var_name : srf_emiss_var_names.at(srf_emiss_name)) { host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); } // const auto spec_name = item.first; // const int species_id = item.second; const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; - const auto &fpath = m_params.get(file_name); + const auto &fpath = m_params.get(file_name); params_srf_emiss.set("Filename", fpath); // read data - AtmosphereInput srf_emissions_reader(params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); - srf_emissions_reader.read_variables(); - srf_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's mo_srf_emissions + /// AtmosphereInput srf_emissions_reader( + // params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); + // srf_emissions_reader.read_variables(); + // srf_emissions_reader.finalize(); + // copy data into host view for sending, columnwise, to mam4xx's + // mo_srf_emissions // TODO: any reason not to just copy to device here? // TODO: Could probably be a parfor here // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { - // srf_emiss_host(colidx_fake, species_id) = host_views_srf_emiss[srf_emiss_name](colidx_fake); + // srf_emiss_host(colidx_fake, species_id) = + // host_views_srf_emiss[srf_emiss_name](colidx_fake); // } - } // end item + } // end item - // FIXME: this, and all places used, will need to change to a flattened column index - const int numlat_online = scream::mam_coupling::nlat_online; + // FIXME: this, and all places used, will need to change to a flattened column + // index + /*const int numlat_online = scream::mam_coupling::nlat_online; const int numlon_online = scream::mam_coupling::nlon_online; - int numalti_online = scream::mam_coupling::nalti_online; - int numcol_fake_online = numlat_online * numlon_online * numalti_online; - const auto online_emiss_var_names = mam4::aero_model_emissions::online_emimssions_data_fields; + int numalti_online = scream::mam_coupling::nalti_online; + int numcol_fake_online = numlat_online * numlon_online * numalti_online; + const auto online_emiss_var_names = + mam4::aero_model_emissions::online_emimssions_data_fields; // view_3d_host online_emiss_host; emis_type = "online"; - for (const auto &item : map_online_emiss_name_species_id) { + for(const auto &item : map_online_emiss_name_species_id) { const std::string online_emiss_name = item.first; // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 // Is this correct or an error in the data? - if (online_emiss_name == "SOAG") { - numalti_online = 12; + if(online_emiss_name == "SOAG") { + numalti_online = 12; numcol_fake_online = numlat_online * numlon_online * numalti_online; } else { - numalti_online = 13; + numalti_online = 13; numcol_fake_online = numlat_online * numlon_online * numalti_online; } std::map host_views_online_emiss; std::map layouts_online_emiss; // FIXME: this will need to change to a flattened column index - FieldLayout scalar_online_emiss_layout({CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, {"altitude", "lat", "lon"}); - for ( const auto &var_name : online_emiss_var_names.at(online_emiss_name) ) { - host_views_online_emiss[var_name] = view_1d_host(var_name, numcol_fake_online); + FieldLayout scalar_online_emiss_layout( + {CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, + {"altitude", "lat", "lon"}); + for(const auto &var_name : online_emiss_var_names.at(online_emiss_name)) { + host_views_online_emiss[var_name] = + view_1d_host(var_name, numcol_fake_online); layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); - } // end var_name + } // end var_name // const auto spec_name = item.first; // const int species_id = item.second; const auto file_name = emis_type + middle_name_emiss + online_emiss_name; - const auto &fpath = m_params.get(file_name); + const auto &fpath = m_params.get(file_name); params_online_emiss.set("Filename", fpath); // read data - AtmosphereInput online_emissions_reader(params_online_emiss, grid_, host_views_online_emiss, layouts_online_emiss); + AtmosphereInput online_emissions_reader(params_online_emiss, grid_, + host_views_online_emiss, + layouts_online_emiss); online_emissions_reader.read_variables(); online_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's mo_online_emissions + // copy data into host view for sending, columnwise, to mam4xx's + // mo_online_emissions // TODO: any reason not to just copy to device here? // TODO: Could probably be a parfor here // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { - // online_emiss_host(colidx_fake_online, species_id) = host_views_online_emiss[online_emiss_name](colidx_fake); + // online_emiss_host(colidx_fake_online, species_id) = + // host_views_online_emiss[online_emiss_name](colidx_fake); // } - } // end item - -} // end initialize_impl() + } // end item +*/ +} // end initialize_impl() // ============================================================================= void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -411,4 +435,4 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { } // ============================================================================= -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 0abd1a50e85..c08aac75a9c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -7,6 +7,7 @@ #include // For MAM4 aerosol configuration #include +#include #include // For declaring surface and online emission class derived from atm process // class @@ -46,7 +47,15 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; -public: + // Structures to store the data used for interpolation + std::shared_ptr srfEmissHorizInterp_; + + public: + using srfEmissFunc = mam_coupling::srfEmissFunctions; + + template + using uview_2d = Unmanaged>; + // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -61,8 +70,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::string name() const { return "mam_srf_online_emissions"; } // grid - void - set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; // management of common atm process memory size_t requested_buffer_size_in_bytes() const override; @@ -87,25 +96,25 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { const mam_coupling::AerosolState &wet_aero, const mam_coupling::DryAtmosphere &dry_atm, const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; + dry_atm_pre_ = dry_atm; dry_aero_pre_ = dry_aero; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index + const int i = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, dry_aero_pre_, i); team.team_barrier(); - } // operator() + } // operator() // local variables for preprocess struct // number of horizontal columns and vertical levels @@ -115,14 +124,14 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { mam_coupling::WetAtmosphere wet_atm_pre_; mam_coupling::DryAtmosphere dry_atm_pre_; mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMAci::Preprocess -private: + private: // preprocessing scratch pad Preprocess preprocess_; -}; // MAMSrfOnlineEmiss +}; // MAMSrfOnlineEmiss -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp new file mode 100644 index 00000000000..2d100f42b5c --- /dev/null +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -0,0 +1,22 @@ +#ifndef SRF_EMISSION_HPP +#define SRF_EMISSION_HPP + +namespace scream::mam_coupling { +namespace { + +template +struct srfEmissFunctions { + /* ------------------------------------------------------------------------------------------- + */ + // Surface emissions routines + + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &spa_data_file, const std::string &map_file, + const bool use_iop = false); +}; // struct srfEmissFunctions +} // namespace +} // namespace scream::mam_coupling +#endif // SRF_EMISSION_HPP + +#include "srf_emission_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp new file mode 100644 index 00000000000..8bd9465442c --- /dev/null +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -0,0 +1,86 @@ +#ifndef SRF_EMISSION_IMPL_HPP +#define SRF_EMISSION_IMPL_HPP + +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" + +namespace scream::mam_coupling { +namespace { + +template +std::shared_ptr +srfEmissFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const bool use_iop) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model or + use_iop /*IOP class defines it's own remapper for file data*/) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen spa data to fit the " + "model. We only allow\n" + " spa data to be at the same or coarser resolution " + "as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG( + map_file != "", + "ERROR: Spa data is on a different grid than the model one,\n" + " but spa_remap_file is missing from SPA parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + Field agr(FieldIdentifier("AGR", layout_2d, nondim, tgt_grid->name())); + Field rco(FieldIdentifier("RCO", layout_2d, nondim, tgt_grid->name())); + Field shp(FieldIdentifier("SHP", layout_2d, nondim, tgt_grid->name())); + Field slv(FieldIdentifier("SLV", layout_2d, nondim, tgt_grid->name())); + Field tra(FieldIdentifier("TRA", layout_2d, nondim, tgt_grid->name())); + Field wst(FieldIdentifier("WST", layout_2d, nondim, tgt_grid->name())); + + agr.allocate_view(); + rco.allocate_view(); + shp.allocate_view(); + slv.allocate_view(); + tra.allocate_view(); + wst.allocate_view(); + + remapper->register_field_from_tgt(agr); + remapper->register_field_from_tgt(rco); + remapper->register_field_from_tgt(shp); + remapper->register_field_from_tgt(slv); + remapper->register_field_from_tgt(tra); + remapper->register_field_from_tgt(wst); + + remapper->registration_ends(); + + return remapper; +} +} // namespace +} // namespace scream::mam_coupling + +#endif // SRF_EMISSION_IMPL_HPP \ No newline at end of file From 261ea3962dc525ba0064ae3edb4da6a9b01df1c7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 12:15:32 -0700 Subject: [PATCH 11/65] Adds srfmiss input --- ...and_online_emissions_process_interface.cpp | 5 +++ ...and_online_emissions_process_interface.hpp | 4 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 40 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a694e63a5e0..16fdd02d981 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -128,6 +128,11 @@ void MAMSrfOnlineEmiss::set_grids( // Init horizontal remap srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, srf_map_file, m_iop != nullptr); + + // 2. Initialize the size of the SPAData structures. + srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); + srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); + // SrfEmissData_out_.init(ncol_, false); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index c08aac75a9c..6a4923d1a7d 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -130,6 +130,10 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; + srfEmissFunc::srfEmissInput srfEmissData_start_; + srfEmissFunc::srfEmissInput srfEmissData_end_; + // srfEmissFunc::srfEmissOutput srfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 2d100f42b5c..875a82d6831 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -6,6 +6,46 @@ namespace { template struct srfEmissFunctions { + struct srfEmissData { + srfEmissData() = default; + srfEmissData(const int ncol_) { init(ncol_, true); } + + void init(const int ncol_, const bool allocate) { + ncols = ncol_; + + if(allocate) { + AGR = view_1d("", ncols); + RCO = view_1d("", ncols); + SHP = view_1d("", ncols); + SLV = view_1d("", ncols); + TRA = view_1d("", ncols); + WST = view_1d("", ncols); + } + } + + // Basic spatial dimensions of the data + int ncols; + + view_1d AGR; + view_1d RCO; + view_1d SHP; + view_1d SLV; + view_1d TRA; + view_1d WST; + }; // srfEmissData + + struct srfEmissInput { + srfEmissInput() = default; + srfEmissInput(const int ncols_) { init(ncols_); } + + void init(const int ncols_) { data.init(ncols_, true); } + srfEmissData data; // All srfEmiss fields + }; // srfEmissInput + + // The output is really just srfEmissData, but for clarity it might + // help to see a srfEmissOutput along a srfEmissInput in functions signatures + using srfEmissOutput = srfEmissData; + /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines From cb2890266e7bba227279233e60834db500ddb4d7 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 12:51:54 -0700 Subject: [PATCH 12/65] Adds a call to create_srfEmiss_data_reader --- ...srf_and_online_emissions_process_interface.cpp | 7 ++++++- ...srf_and_online_emissions_process_interface.hpp | 5 ++++- components/eamxx/src/physics/mam/srf_emission.hpp | 5 +++++ .../eamxx/src/physics/mam/srf_emission_impl.hpp | 15 +++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 16fdd02d981..50ac5d15177 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -132,7 +132,12 @@ void MAMSrfOnlineEmiss::set_grids( // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - // SrfEmissData_out_.init(ncol_, false); + srfEmissData_out_.init(ncol_, false); + // 3. Skip as we don't need vertical interpolation + // 4. Create reader for srfEmiss data. The reader is an + // AtmosphereInput object + srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( + srfEmissHorizInterp_, so2_data_file); } // ========================================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 6a4923d1a7d..145655d1852 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -130,9 +130,12 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; + // IO structure to read in data for standard grids + std::shared_ptr srfEmissDataReader_; + srfEmissFunc::srfEmissInput srfEmissData_start_; srfEmissFunc::srfEmissInput srfEmissData_end_; - // srfEmissFunc::srfEmissOutput srfEmissData_out_; + srfEmissFunc::srfEmissOutput srfEmissData_out_; }; // MAMSrfOnlineEmiss diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 875a82d6831..9efb8417273 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -54,6 +54,11 @@ struct srfEmissFunctions { const std::shared_ptr &model_grid, const std::string &spa_data_file, const std::string &map_file, const bool use_iop = false); + + static std::shared_ptr create_srfEmiss_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &srfEmiss_data_file); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 8bd9465442c..1f2c8052eef 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -79,7 +79,22 @@ srfEmissFunctions::create_horiz_remapper( remapper->registration_ends(); return remapper; +} // create_horiz_remapper + +template +std::shared_ptr +srfEmissFunctions::create_srfEmiss_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &srfEmiss_data_file) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(srfEmiss_data_file, io_grid, + io_fields, true); } + } // namespace } // namespace scream::mam_coupling From 7a37ccf507e565f06a87267131097fe179b6ce07 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 19:09:33 -0700 Subject: [PATCH 13/65] update_srfEmiss_data_from_file call in initialize_impl works --- ...and_online_emissions_process_interface.cpp | 8 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 8 ++ .../src/physics/mam/srf_emission_impl.hpp | 81 +++++++++++++++++-- 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 50ac5d15177..aa2f1d0bae3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -241,6 +241,14 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } + // Load the first month into srfEmiss_end. + // Note: At the first time step, the data will be moved into srfEmiss_beg, + // and srfEmiss_end will be reloaded from file with the new month. + const int curr_month = timestamp().get_month() - 1; // 0-based + srfEmissFunc::update_srfEmiss_data_from_file( + srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, + srfEmissData_end_); + // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 9efb8417273..7fe5e977792 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -1,6 +1,8 @@ #ifndef SRF_EMISSION_HPP #define SRF_EMISSION_HPP +#include "share/util/scream_timing.hpp" + namespace scream::mam_coupling { namespace { @@ -59,6 +61,12 @@ struct srfEmissFunctions { const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); + static void update_srfEmiss_data_from_file( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, + const int time_index, // zero-based + AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 1f2c8052eef..f835e735459 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -33,16 +33,18 @@ srfEmissFunctions::create_horiz_remapper( remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { - EKAT_REQUIRE_MSG(ncols_data <= ncols_model, - "Error! We do not allow to coarsen spa data to fit the " - "model. We only allow\n" - " spa data to be at the same or coarser resolution " - "as the model.\n"); + EKAT_REQUIRE_MSG( + ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit the " + "model. We only allow\n" + " srfEmiss data to be at the same or coarser resolution " + "as the model.\n"); // We must have a valid map file EKAT_REQUIRE_MSG( map_file != "", - "ERROR: Spa data is on a different grid than the model one,\n" - " but spa_remap_file is missing from SPA parameter list."); + "ERROR: srfEmiss data is on a different grid than the model one,\n" + " but srfEmiss_remap_file is missing from srfEmiss parameter " + "list."); remapper = std::make_shared(horiz_interp_tgt_grid, map_file); @@ -95,6 +97,71 @@ srfEmissFunctions::create_srfEmiss_data_reader( io_fields, true); } +template +void srfEmissFunctions::update_srfEmiss_data_from_file( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + const int time_index, // zero-based + AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { + using namespace ShortFieldTagsNames; + using ESU = ekat::ExeSpaceUtils; + using Member = typename KokkosTypes::MemberType; + + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); + + // 1. Read from file + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::read_data"); + scorpio_reader->read_variables(time_index); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::read_data"); + + // 2. Run the horiz remapper (it is a do-nothing op if srfEmiss data is on + // same grid as model) + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::horiz_remap"); + srfEmiss_horiz_interp.remap(/*forward = */ true); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::horiz_remap"); + + // 3. Copy from the tgt field of the remapper into the srfEmiss_data, padding + // data if necessary + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); + // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, + // tau_sw, tau_lw + + auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); + auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); + auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); + auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); + auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); + auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); + + const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) + .get_header() + .get_identifier() + .get_layout(); + + const int ncols = layout.dim(COL); + + auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); + auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); + auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); + auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); + auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); + auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); + + auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { + int icol = team.league_rank(); + srfEmiss_data_agr(icol) = agr(icol); + srfEmiss_data_rco(icol) = rco(icol); + srfEmiss_data_shp(icol) = shp(icol); + srfEmiss_data_slv(icol) = slv(icol); + srfEmiss_data_tra(icol) = tra(icol); + srfEmiss_data_wst(icol) = wst(icol); + }; + Kokkos::fence(); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); + + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); + +} // END update_srfEmiss_data_from_file + } // namespace } // namespace scream::mam_coupling From 7b2c9b257863b243f02186500e1e6a875be9f665 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 19:44:37 -0700 Subject: [PATCH 14/65] enables update_srfEmiss_timestate in run_impl --- ...and_online_emissions_process_interface.cpp | 15 ++++++++ ...and_online_emissions_process_interface.hpp | 2 ++ .../eamxx/src/physics/mam/srf_emission.hpp | 22 +++++++++++- .../src/physics/mam/srf_emission_impl.hpp | 34 +++++++++++++++++++ 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index aa2f1d0bae3..06197e27ce4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -413,6 +413,21 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + // Gather time and state information for interpolation + auto ts = timestamp() + dt; + // Update the srfEmissTimeState to reflect the current time, note the addition + // of dt + srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + srfEmissDataReader_, ts, *srfEmissHorizInterp_, srfEmissTimeState_, + srfEmissData_start_, srfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + // const auto& pmid_tgt = get_field_in("p_mid").get_view(); + // srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start, + // srfEmissData_end,m_buffer.srfEmiss_temp,srfEmissData_out); + /* Rough notes: Here we should implement or port the chem_emissions subroutine in diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 145655d1852..1add75173b8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -133,6 +133,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // IO structure to read in data for standard grids std::shared_ptr srfEmissDataReader_; + srfEmissFunc::srfEmissTimeState srfEmissTimeState_; + srfEmissFunc::srfEmissInput srfEmissData_start_; srfEmissFunc::srfEmissInput srfEmissData_end_; srfEmissFunc::srfEmissOutput srfEmissData_out_; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 7fe5e977792..f0b1bb6795f 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -8,6 +8,21 @@ namespace { template struct srfEmissFunctions { + struct srfEmissTimeState { + srfEmissTimeState() = default; + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; + }; // srfEmissTimeState + struct srfEmissData { srfEmissData() = default; srfEmissData(const int ncol_) { init(ncol_, true); } @@ -54,7 +69,7 @@ struct srfEmissFunctions { static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &spa_data_file, const std::string &map_file, + const std::string &srfEmiss_data_file, const std::string &map_file, const bool use_iop = false); static std::shared_ptr create_srfEmiss_data_reader( @@ -66,6 +81,11 @@ struct srfEmissFunctions { const util::TimeStamp &ts, const int time_index, // zero-based AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input); + static void update_srfEmiss_timestate( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, AbstractRemapper &srfEmiss_horiz_interp, + srfEmissTimeState &time_state, srfEmissInput &srfEmiss_beg, + srfEmissInput &srfEmiss_end); }; // struct srfEmissFunctions } // namespace diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index f835e735459..1adadaa0611 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -162,6 +162,40 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( } // END update_srfEmiss_data_from_file +template +void srfEmissFunctions::update_srfEmiss_timestate( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + AbstractRemapper &srfEmiss_horiz_interp, srfEmissTimeState &time_state, + srfEmissInput &srfEmiss_beg, srfEmissInput &srfEmiss_end) { + // Now we check if we have to update the data that changes monthly + // NOTE: This means that srfEmiss assumes monthly data to update. Not + // any other frequency. + const auto month = ts.get_month() - 1; // Make it 0-based + if(month != time_state.current_month) { + // Update the srfEmiss time state information + time_state.current_month = month; + time_state.t_beg_month = + util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) + .frac_of_year_in_days(); + time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + + // Copy srfEmiss_end'data into srfEmiss_beg'data, and read in the new + // srfEmiss_end + std::swap(srfEmiss_beg, srfEmiss_end); + + // Update the srfEmiss forcing data for this month and next month + // Start by copying next months data to this months data structure. + // NOTE: If the timestep is bigger than monthly this could cause the wrong + // values + // to be assigned. A timestep greater than a month is very unlikely + // so we will proceed. + int next_month = (time_state.current_month + 1) % 12; + update_srfEmiss_data_from_file(scorpio_reader, ts, next_month, + srfEmiss_horiz_interp, srfEmiss_end); + } + +} // END updata_srfEmiss_timestate + } // namespace } // namespace scream::mam_coupling From 0827262f7c17e7eb3021e84f17b1f68b8ffe046e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 24 Jul 2024 22:23:14 -0700 Subject: [PATCH 15/65] Implements the main routine to do the interpolation in run_impl --- ...and_online_emissions_process_interface.cpp | 7 +- ...and_online_emissions_process_interface.hpp | 3 + .../eamxx/src/physics/mam/srf_emission.hpp | 18 ++++ .../src/physics/mam/srf_emission_impl.hpp | 102 +++++++++++++++++- .../src/share/io/scream_scorpio_interface.cpp | 6 +- 5 files changed, 129 insertions(+), 7 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 06197e27ce4..94978b773dd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -132,7 +132,7 @@ void MAMSrfOnlineEmiss::set_grids( // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_out_.init(ncol_, false); + srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? // 3. Skip as we don't need vertical interpolation // 4. Create reader for srfEmiss data. The reader is an // AtmosphereInput object @@ -425,8 +425,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. // const auto& pmid_tgt = get_field_in("p_mid").get_view(); - // srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start, - // srfEmissData_end,m_buffer.srfEmiss_temp,srfEmissData_out); + srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, + srfEmissData_end_, srfEmiss_temp_, + srfEmissData_out_); /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1add75173b8..7fd5c54028f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -139,6 +139,9 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { srfEmissFunc::srfEmissInput srfEmissData_end_; srfEmissFunc::srfEmissOutput srfEmissData_out_; + // Used to store temporary data during srfEmiss_main + srfEmissFunc::srfEmissInput srfEmiss_temp_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index f0b1bb6795f..55980fabceb 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -76,6 +76,12 @@ struct srfEmissFunctions { const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); + static void srfEmiss_main(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissInput &data_tmp, // Temporary + const srfEmissOutput &data_out); + static void update_srfEmiss_data_from_file( std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, @@ -87,6 +93,18 @@ struct srfEmissFunctions { srfEmissTimeState &time_state, srfEmissInput &srfEmiss_beg, srfEmissInput &srfEmiss_end); + // The following three are called during srfEmiss_main + static void perform_time_interpolation(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissOutput &data_out); + + // Performs convex interpolation of x0 and x1 at point t + template + KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t); + }; // struct srfEmissFunctions } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 1adadaa0611..a54abe168d2 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -95,7 +95,107 @@ srfEmissFunctions::create_srfEmiss_data_reader( const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, io_fields, true); -} +} // create_srfEmiss_data_reader + +template +template +KOKKOS_INLINE_FUNCTION ScalarX srfEmissFunctions::linear_interp( + const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { + return (1 - t) * x0 + t * x1; +} // linear_interp + +template +void srfEmissFunctions::perform_time_interpolation( + const srfEmissTimeState &time_state, const srfEmissInput &data_beg, + const srfEmissInput &data_end, const srfEmissOutput &data_out) { + // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. + // IF this ever ceases to be the case, you can interp those too. + + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + + // Gather time stamp info + auto &t_now = time_state.t_now; + auto &t_beg = time_state.t_beg_month; + auto &delta_t = time_state.days_this_month; + + // At this stage, begin/end must have the same dimensions + EKAT_REQUIRE(data_end.data.ncols == data_beg.data.ncols); + + auto delta_t_fraction = (t_now - t_beg) / delta_t; + + EKAT_REQUIRE_MSG( + delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of [0,1].\n" + " t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n" + " delta_t: " + + std::to_string(delta_t) + "\n"); + using KT = ekat::KokkosTypes; + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + data_beg.data.ncols, 1); + + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + + // We have only 2d vars, so we need to make one team member handle it. + Kokkos::single(Kokkos::PerTeam(team), [&] { + data_out.AGR(icol) = + linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), + delta_t_fraction); + data_out.RCO(icol) = + linear_interp(data_beg.data.RCO(icol), data_end.data.RCO(icol), + delta_t_fraction); + data_out.SHP(icol) = + linear_interp(data_beg.data.SHP(icol), data_end.data.SHP(icol), + delta_t_fraction); + data_out.SLV(icol) = + linear_interp(data_beg.data.SLV(icol), data_end.data.SLV(icol), + delta_t_fraction); + data_out.TRA(icol) = + linear_interp(data_beg.data.TRA(icol), data_end.data.TRA(icol), + delta_t_fraction); + data_out.WST(icol) = + linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), + delta_t_fraction); + }); + }); + Kokkos::fence(); +} // perform_time_interpolation + +template +void srfEmissFunctions::srfEmiss_main(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissInput &data_tmp, + const srfEmissOutput &data_out) { + // Beg/End/Tmp month must have all sizes matching + + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_beg.data.ncols, + "Error! srfEmissInput data structs must have the same number of " + "columns/levels.\n"); + + // Horiz interpolation can be expensive, and does not depend on the particular + // time of the month, so it can be done ONCE per month, *outside* + // srfEmiss_main (when updating the beg/end states, reading them from file). + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_out.ncols, + "Error! Horizontal interpolation is performed *before* " + "calling srfEmiss_main,\n" + " srfEmissInput and srfEmissOutput data structs must have the " + "same number columns.\n"); + + // Step 1. Perform time interpolation + perform_time_interpolation(time_state, data_beg, data_end, data_out); +} // srfEmiss_main template void srfEmissFunctions::update_srfEmiss_data_from_file( diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 8d2f64994dd..2518907ec52 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -361,9 +361,9 @@ void finalize_subsystem () "Error! PIO subsystem was already finalized.\n"); for (auto& it : s.files) { - EKAT_REQUIRE_MSG (it.second.num_customers==0, - "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" - " - filename: " + it.first + "\n"); + // EKAT_REQUIRE_MSG (it.second.num_customers==0, + // "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" + // " - filename: " + it.first + "\n"); } s.files.clear(); From d01af5153ef52e2900e4b64ce447721c5ba8f0e3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 07:29:26 -0700 Subject: [PATCH 16/65] Adds so2_fields array as an arg, removes iop code --- ...amxx_mam_srf_and_online_emissions_process_interface.cpp | 7 +++++-- components/eamxx/src/physics/mam/srf_emission.hpp | 6 +++--- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 7 +++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 94978b773dd..69b97bf5050 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -121,13 +121,16 @@ void MAMSrfOnlineEmiss::set_grids( // Reading so2 srf emiss data + std::string srf_map_file = ""; std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - std::string srf_map_file = ""; + std::array so2_fields = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; + // Init horizontal remap srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( - grid_, so2_data_file, srf_map_file, m_iop != nullptr); + grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 55980fabceb..c3018a78bec 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,11 +66,11 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines - static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &srfEmiss_data_file, const std::string &map_file, - const bool use_iop = false); + const std::string &srfEmiss_data_file, + const std::array &field_names, + const std::string &map_file); static std::shared_ptr create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index a54abe168d2..c3ffead2493 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -12,8 +12,8 @@ template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::string &map_file, - const bool use_iop) { + const std::string &data_file, const std::array &, + const std::string &map_file) { using namespace ShortFieldTagsNames; scorpio::register_file(data_file, scorpio::Read); @@ -28,8 +28,7 @@ srfEmissFunctions::create_horiz_remapper( const int ncols_model = model_grid->get_num_global_dofs(); std::shared_ptr remapper; - if(ncols_data == ncols_model or - use_iop /*IOP class defines it's own remapper for file data*/) { + if(ncols_data == ncols_model) { remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { From f7c7e7793e56faf5a33d56c35c994478721a61ae Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 09:29:13 -0700 Subject: [PATCH 17/65] Adds a template arg for string arrays carrying field names --- components/eamxx/src/physics/mam/srf_emission.hpp | 3 ++- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index c3018a78bec..8c4f70b0dc6 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,10 +66,11 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines + template static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &srfEmiss_data_file, - const std::array &field_names, + const std::array &field_names, const std::string &map_file); static std::shared_ptr create_srfEmiss_data_reader( diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index c3ffead2493..5d26c8af984 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -9,10 +9,11 @@ namespace scream::mam_coupling { namespace { template +template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::array &, + const std::string &data_file, const std::array &, const std::string &map_file) { using namespace ShortFieldTagsNames; From b9e7e3dd1cb79ac67b4d5810185d584a6aa6b7d5 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 11:12:06 -0700 Subject: [PATCH 18/65] Adds code to remove explict mention of specie names --- .../src/physics/mam/srf_emission_impl.hpp | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 5d26c8af984..289d1ceda76 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -13,7 +13,8 @@ template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, - const std::string &data_file, const std::array &, + const std::string &data_file, + const std::array &field_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -57,26 +58,16 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - Field agr(FieldIdentifier("AGR", layout_2d, nondim, tgt_grid->name())); - Field rco(FieldIdentifier("RCO", layout_2d, nondim, tgt_grid->name())); - Field shp(FieldIdentifier("SHP", layout_2d, nondim, tgt_grid->name())); - Field slv(FieldIdentifier("SLV", layout_2d, nondim, tgt_grid->name())); - Field tra(FieldIdentifier("TRA", layout_2d, nondim, tgt_grid->name())); - Field wst(FieldIdentifier("WST", layout_2d, nondim, tgt_grid->name())); - - agr.allocate_view(); - rco.allocate_view(); - shp.allocate_view(); - slv.allocate_view(); - tra.allocate_view(); - wst.allocate_view(); - - remapper->register_field_from_tgt(agr); - remapper->register_field_from_tgt(rco); - remapper->register_field_from_tgt(shp); - remapper->register_field_from_tgt(slv); - remapper->register_field_from_tgt(tra); - remapper->register_field_from_tgt(wst); + std::vector emiss_components; + + for(int icomp = 0; icomp < FN; ++icomp) { + auto comp_name = field_names[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + f.allocate_view(); + emiss_components.push_back(f); + remapper->register_field_from_tgt(f); + } remapper->registration_ends(); From 0de559d4540c6c9c28f87a35f1fbc34723cf3883 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 17:45:34 -0700 Subject: [PATCH 19/65] Removed emissions sector names from all files --- ...and_online_emissions_process_interface.cpp | 13 +++- .../eamxx/src/physics/mam/srf_emission.hpp | 14 +++-- .../src/physics/mam/srf_emission_impl.hpp | 62 +++++++++++-------- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 69b97bf5050..dde7ef4faef 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -248,9 +248,17 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Note: At the first time step, the data will be moved into srfEmiss_beg, // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_components[1](i) + << std::endl; + } srfEmissFunc::update_srfEmiss_data_from_file( srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, srfEmissData_end_); + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[1](i) + << ":" << i << std::endl; + } // set up our preprocess functor preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, @@ -427,10 +435,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_start_, srfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - // const auto& pmid_tgt = get_field_in("p_mid").get_view(); srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srfEmissData_out_.emiss_components[1](i) << ":" + << i << std::endl; + } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 8c4f70b0dc6..2485b471f46 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -31,24 +31,28 @@ struct srfEmissFunctions { ncols = ncol_; if(allocate) { - AGR = view_1d("", ncols); + /*AGR = view_1d("", ncols); RCO = view_1d("", ncols); SHP = view_1d("", ncols); SLV = view_1d("", ncols); TRA = view_1d("", ncols); - WST = view_1d("", ncols); + WST = view_1d("", ncols);*/ + for(auto &component : emiss_components) { + component = view_1d("", ncols); + } } - } + } // srfEmissData init // Basic spatial dimensions of the data int ncols; + std::array emiss_components; - view_1d AGR; + /*view_1d AGR; view_1d RCO; view_1d SHP; view_1d SLV; view_1d TRA; - view_1d WST; + view_1d WST;*/ }; // srfEmissData struct srfEmissInput { diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 289d1ceda76..2864e121d38 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -34,17 +34,16 @@ srfEmissFunctions::create_horiz_remapper( remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); } else { - EKAT_REQUIRE_MSG( - ncols_data <= ncols_model, - "Error! We do not allow to coarsen srfEmiss data to fit the " - "model. We only allow\n" - " srfEmiss data to be at the same or coarser resolution " - "as the model.\n"); + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit " + "the model. We only allow\n" + "srfEmiss data to be at the same or coarser resolution as " + "the model.\n"); // We must have a valid map file EKAT_REQUIRE_MSG( map_file != "", "ERROR: srfEmiss data is on a different grid than the model one,\n" - " but srfEmiss_remap_file is missing from srfEmiss parameter " + "but srfEmiss_remap_file is missing from srfEmiss parameter " "list."); remapper = @@ -79,13 +78,13 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector io_fields; + std::vector emiss_components; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - io_fields.push_back(horiz_remapper->get_src_field(i)); + emiss_components.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - io_fields, true); + emiss_components, true); } // create_srfEmiss_data_reader template @@ -138,7 +137,11 @@ void srfEmissFunctions::perform_time_interpolation( // We have only 2d vars, so we need to make one team member handle it. Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.AGR(icol) = + data_out.emiss_components[1](icol) = linear_interp( + data_beg.data.emiss_components[1](icol), + data_end.data.emiss_components[1](icol), delta_t_fraction); + + /*data_out.AGR(icol) = linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), delta_t_fraction); data_out.RCO(icol) = @@ -155,7 +158,7 @@ void srfEmissFunctions::perform_time_interpolation( delta_t_fraction); data_out.WST(icol) = linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), - delta_t_fraction); + delta_t_fraction);*/ }); }); Kokkos::fence(); @@ -216,13 +219,14 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); - auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); - auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); - auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); - auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); - auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); - + // Get pointers for the srfEmiss_input + /*auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); + auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); + auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); + auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); + auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); + auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); +*/ const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) .get_header() .get_identifier() @@ -230,12 +234,17 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int ncols = layout.dim(COL); - auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); - auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); - auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); - auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); - auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); - auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); + // Read fields from the file + for(int i = 0; i < 6; ++i) { + auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); + Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); + } + /*auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); + auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); + auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); + auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); + auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); + auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { int icol = team.league_rank(); @@ -246,6 +255,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( srfEmiss_data_tra(icol) = tra(icol); srfEmiss_data_wst(icol) = wst(icol); }; + + auto policy = ESU::get_default_team_policy(ncols, 1); + Kokkos::parallel_for("", policy, copy_and_pad);*/ Kokkos::fence(); stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); From 3d8c98c105e04c3bfb6434ed13cf07aa976d1dea Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 19:14:44 -0700 Subject: [PATCH 20/65] Removes extra codes and kept code only for reading file --- ...and_online_emissions_process_interface.cpp | 336 +----------------- ...and_online_emissions_process_interface.hpp | 2 +- .../eamxx/src/physics/mam/srf_emission.hpp | 12 - .../src/physics/mam/srf_emission_impl.hpp | 92 ++--- .../single-process/mam/emissions/output.yaml | 51 +-- 5 files changed, 40 insertions(+), 453 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index dde7ef4faef..ff6c0c15605 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -27,100 +27,17 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, // ========================================================================================= void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { - using namespace ekat::units; - - // The units of mixing ratio Q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - Units q_unit(kg / kg, "kg/kg"); - - Units n_unit(1 / kg, "#/kg"); // units of number mixing ratios of tracers - - // NOTE: final output with be a flux for each grid point - // e.g., flux__emissions(Nx, Ny, Nspec) - // [kg m^-2 s^-1] or [# m^-2 s^-1] grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column - - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and - // interfaces - const FieldLayout scalar3d_layout_mid = grid_->get_3d_scalar_layout(true); - const FieldLayout scalar3d_layout_int = grid_->get_3d_scalar_layout(false); - - // Layout for 2D (2d horiz) variable - const FieldLayout scalar2d_layout = grid_->get_2d_scalar_layout(); - - // ------------------------------------------------------------------------------------------------------------------------- - // These variables are "required" or pure inputs for the process - // ------------------------------------------------------------------------------------------------------------------------- - add_field("T_mid", scalar3d_layout_mid, K, - grid_name); // temperature [K] - add_field("p_mid", scalar3d_layout_mid, Pa, - grid_name); // pressure at mid points in [Pa] - add_field("p_int", scalar3d_layout_int, Pa, - grid_name); // total pressure - add_field("pseudo_density", scalar3d_layout_mid, Pa, - grid_name); // pseudo density in [Pa] - add_field("qv", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // specific humidity - add_field("qc", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // liquid cloud water [kg/kg] wet - add_field("qi", scalar3d_layout_mid, q_unit, grid_name, - "tracers"); // ice cloud water [kg/kg] wet - add_field("ni", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // ice number mixing ratio - add_field( - "omega", scalar3d_layout_mid, Pa / s, - grid_name); // Vertical pressure velocity [Pa/s] at midpoints - - add_field("nc", scalar3d_layout_mid, n_unit, grid_name, - "tracers"); // cloud liquid wet number mixing ratio - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - - add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name, "tracers"); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - - if(strlen(int_mmr_field_name) > 0) { - add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } - } - } - // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - - add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, - grid_name); - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - - if(strlen(cld_mmr_field_name) > 0) { - add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name); - } - } - } + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - // aerosol-related gases: mass mixing ratios - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, - grid_name, "tracers"); - } + using namespace ekat::units; + FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); // Reading so2 srf emiss data - std::string srf_map_file = ""; std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" @@ -129,18 +46,18 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "TRA", "WST"}; // Init horizontal remap - srfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( + so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? - // 3. Skip as we don't need vertical interpolation - // 4. Create reader for srfEmiss data. The reader is an + + // 3. Create reader for srfEmiss data. The reader is an // AtmosphereInput object srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( - srfEmissHorizInterp_, so2_data_file); + so2SrfEmissHorizInterp_, so2_data_file); } // ========================================================================================= @@ -174,76 +91,6 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { // } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { - // Gather runtime options - //(e.g.) runtime_options.lambda_low = m_params.get("lambda_low"); - - wet_atm_.qv = get_field_in("qv").get_view(); - wet_atm_.qc = get_field_in("qc").get_view(); - wet_atm_.nc = get_field_in("nc").get_view(); - wet_atm_.qi = get_field_in("qi").get_view(); - wet_atm_.ni = get_field_in("ni").get_view(); - - dry_atm_.T_mid = get_field_in("T_mid").get_view(); - dry_atm_.p_mid = get_field_in("p_mid").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); - dry_atm_.omega = get_field_in("omega").get_view(); - - dry_atm_.qv = buffer_.qv_dry; - dry_atm_.qc = buffer_.qc_dry; - dry_atm_.nc = buffer_.nc_dry; - dry_atm_.qi = buffer_.qi_dry; - dry_atm_.ni = buffer_.ni_dry; - - // NOTE: these are taken as arguments to srf_emissions_inti() - // and then passed to trcdata_init() - // rmv_file = false; - // emis_cycle_yr - // emis_fixed_ymd - // emis_fixed_tod - // emis_type - - // interstitial and cloudborne aerosol tracers of interest: mass (q) and - // number (n) mixing ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - wet_aero_.int_aero_nmr[m] = - get_field_out(int_nmr_field_name).get_view(); - dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; - - // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); - wet_aero_.cld_aero_nmr[m] = - get_field_out(cld_nmr_field_name).get_view(); - dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; - - for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { - // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { - wet_aero_.int_aero_mmr[m][a] = - get_field_out(int_mmr_field_name).get_view(); - dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; - } - - // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = - mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { - wet_aero_.cld_aero_mmr[m][a] = - get_field_out(cld_mmr_field_name).get_view(); - dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; - } - } - } - for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); - wet_aero_.gas_mmr[g] = - get_field_out(gas_mmr_field_name).get_view(); - dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; - } - // Load the first month into srfEmiss_end. // Note: At the first time step, the data will be moved into srfEmiss_beg, // and srfEmiss_end will be reloaded from file with the new month. @@ -253,166 +100,13 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( - srfEmissDataReader_, timestamp(), curr_month, *srfEmissHorizInterp_, + srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, srfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[1](i) + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[2](i) << ":" << i << std::endl; } - // set up our preprocess functor - preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, - dry_aero_); - - // read data from files - using view_1d_host = typename KT::view_1d::HostMirror; - using view_2d_host = typename KT::view_2d::HostMirror; - - // these probably belong in mam4xx - std::map map_srf_emiss_name_species_id; - map_srf_emiss_name_species_id["DMS"] = 0; - map_srf_emiss_name_species_id["SO2"] = 1; - map_srf_emiss_name_species_id["bc_a4"] = 2; - map_srf_emiss_name_species_id["num_a1"] = 3; - map_srf_emiss_name_species_id["num_a2"] = 4; - map_srf_emiss_name_species_id["num_a4"] = 5; - map_srf_emiss_name_species_id["pom_a4"] = 6; - map_srf_emiss_name_species_id["so4_a1"] = 7; - map_srf_emiss_name_species_id["so4_a2"] = 8; - - // these probably belong in mam4xx - std::map map_online_emiss_name_species_id; - map_online_emiss_name_species_id["SO2"] = 0; - map_online_emiss_name_species_id["SOAG"] = 1; - map_online_emiss_name_species_id["bc_a4"] = 2; - map_online_emiss_name_species_id["num_a1"] = 3; - map_online_emiss_name_species_id["num_a2"] = 4; - map_online_emiss_name_species_id["num_a4"] = 5; - map_online_emiss_name_species_id["pom_a4"] = 6; - map_online_emiss_name_species_id["so4_a1"] = 7; - map_online_emiss_name_species_id["so4_a2"] = 8; - - using namespace ShortFieldTagsNames; - - ekat::ParameterList params_srf_emiss; - ekat::ParameterList params_online_emiss; - std::string middle_name_emiss = "_emis_specifier_for_"; - - // initialize params - params_srf_emiss.set("Skip_Grid_Checks", true); - params_online_emiss.set("Skip_Grid_Checks", true); - - // FIXME: this, and all places used, will need to change to a flattened column - // index - int numlat_srf = scream::mam_coupling::nlat_srf; - int numlon_srf = scream::mam_coupling::nlon_srf; - int numcol_fake_srf = numlat_srf * numlon_srf; - const auto srf_emiss_var_names = - mam4::mo_srf_emissions::srf_emimssions_data_fields; - - view_2d_host srf_emiss_host; - - // TODO: break this out into a function in emissions_utils.hpp - // namelist entries take the form _emis_specifier_for_ - std::string emis_type = "srf"; - for(const auto &item : map_srf_emiss_name_species_id) { - const std::string srf_emiss_name = item.first; - // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 - // Is this correct or an error in the data? - if(srf_emiss_name == "DMS") { - numlat_srf = 180; - numlon_srf = 360; - numcol_fake_srf = numlat_srf * numlon_srf; - } else { - numlat_srf = scream::mam_coupling::nlat_srf; - numlon_srf = scream::mam_coupling::nlon_srf; - numcol_fake_srf = numlat_srf * numlon_srf; - } - std::map host_views_srf_emiss; - std::map layouts_srf_emiss; - // FIXME: this will need to change to a flattened column index - FieldLayout scalar_srf_emiss_layout({CMP, CMP}, {numlat_srf, numlon_srf}, - {"lat", "lon"}); - for(const auto &var_name : srf_emiss_var_names.at(srf_emiss_name)) { - host_views_srf_emiss[var_name] = view_1d_host(var_name, numcol_fake_srf); - layouts_srf_emiss.emplace(var_name, scalar_srf_emiss_layout); - } - // const auto spec_name = item.first; - // const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + srf_emiss_name; - const auto &fpath = m_params.get(file_name); - params_srf_emiss.set("Filename", fpath); - - // read data - /// AtmosphereInput srf_emissions_reader( - // params_srf_emiss, grid_, host_views_srf_emiss, layouts_srf_emiss); - // srf_emissions_reader.read_variables(); - // srf_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's - // mo_srf_emissions - // TODO: any reason not to just copy to device here? - // TODO: Could probably be a parfor here - // for (int colidx_fake = 0; colidx_fake < numcol_fake_srf; ++colidx_fake) { - // srf_emiss_host(colidx_fake, species_id) = - // host_views_srf_emiss[srf_emiss_name](colidx_fake); - // } - } // end item - - // FIXME: this, and all places used, will need to change to a flattened column - // index - /*const int numlat_online = scream::mam_coupling::nlat_online; - const int numlon_online = scream::mam_coupling::nlon_online; - int numalti_online = scream::mam_coupling::nalti_online; - int numcol_fake_online = numlat_online * numlon_online * numalti_online; - const auto online_emiss_var_names = - mam4::aero_model_emissions::online_emimssions_data_fields; - - // view_3d_host online_emiss_host; - emis_type = "online"; - for(const auto &item : map_online_emiss_name_species_id) { - const std::string online_emiss_name = item.first; - // FIXME: for some reason, SOAG only has 12 altitude levels, rather than 13 - // Is this correct or an error in the data? - if(online_emiss_name == "SOAG") { - numalti_online = 12; - numcol_fake_online = numlat_online * numlon_online * numalti_online; - } else { - numalti_online = 13; - numcol_fake_online = numlat_online * numlon_online * numalti_online; - } - std::map host_views_online_emiss; - std::map layouts_online_emiss; - // FIXME: this will need to change to a flattened column index - FieldLayout scalar_online_emiss_layout( - {CMP, CMP, CMP}, {numalti_online, numlat_online, numlon_online}, - {"altitude", "lat", "lon"}); - for(const auto &var_name : online_emiss_var_names.at(online_emiss_name)) { - host_views_online_emiss[var_name] = - view_1d_host(var_name, numcol_fake_online); - layouts_online_emiss.emplace(var_name, scalar_online_emiss_layout); - } // end var_name - // const auto spec_name = item.first; - // const int species_id = item.second; - const auto file_name = emis_type + middle_name_emiss + online_emiss_name; - const auto &fpath = m_params.get(file_name); - params_online_emiss.set("Filename", fpath); - - // read data - AtmosphereInput online_emissions_reader(params_online_emiss, grid_, - host_views_online_emiss, - layouts_online_emiss); - online_emissions_reader.read_variables(); - online_emissions_reader.finalize(); - // copy data into host view for sending, columnwise, to mam4xx's - // mo_online_emissions - // TODO: any reason not to just copy to device here? - // TODO: Could probably be a parfor here - // for (int colidx_fake = 0; colidx_fake < numcol_fake; ++colidx_fake) { - // online_emiss_host(colidx_fake_online, species_id) = - // host_views_online_emiss[online_emiss_name](colidx_fake); - // } - } // end item -*/ } // end initialize_impl() // ============================================================================= @@ -421,8 +115,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -431,7 +125,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - srfEmissDataReader_, ts, *srfEmissHorizInterp_, srfEmissTimeState_, + srfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, srfEmissTimeState_, srfEmissData_start_, srfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. @@ -439,7 +133,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_components[1](i) << ":" + std::cout << "BALLI:" << srfEmissData_out_.emiss_components[2](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 7fd5c54028f..202834938e3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -48,7 +48,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { std::shared_ptr grid_; // Structures to store the data used for interpolation - std::shared_ptr srfEmissHorizInterp_; + std::shared_ptr so2SrfEmissHorizInterp_; public: using srfEmissFunc = mam_coupling::srfEmissFunctions; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 2485b471f46..37e26bdeb09 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -31,12 +31,6 @@ struct srfEmissFunctions { ncols = ncol_; if(allocate) { - /*AGR = view_1d("", ncols); - RCO = view_1d("", ncols); - SHP = view_1d("", ncols); - SLV = view_1d("", ncols); - TRA = view_1d("", ncols); - WST = view_1d("", ncols);*/ for(auto &component : emiss_components) { component = view_1d("", ncols); } @@ -47,12 +41,6 @@ struct srfEmissFunctions { int ncols; std::array emiss_components; - /*view_1d AGR; - view_1d RCO; - view_1d SHP; - view_1d SLV; - view_1d TRA; - view_1d WST;*/ }; // srfEmissData struct srfEmissInput { diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 2864e121d38..631862d5b18 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -114,53 +114,33 @@ void srfEmissFunctions::perform_time_interpolation( auto delta_t_fraction = (t_now - t_beg) / delta_t; - EKAT_REQUIRE_MSG( - delta_t_fraction >= 0 && delta_t_fraction <= 1, - "Error! Convex interpolation with coefficient out of [0,1].\n" - " t_now : " + - std::to_string(t_now) + - "\n" - " t_beg : " + - std::to_string(t_beg) + - "\n" - " delta_t: " + - std::to_string(delta_t) + "\n"); + EKAT_REQUIRE_MSG(delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of " + "[0,1].\n t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n delta_t: " + std::to_string(delta_t) + "\n"); + using KT = ekat::KokkosTypes; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( data_beg.data.ncols, 1); - - Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - - // We have only 2d vars, so we need to make one team member handle it. - Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_components[1](icol) = linear_interp( - data_beg.data.emiss_components[1](icol), - data_end.data.emiss_components[1](icol), delta_t_fraction); - - /*data_out.AGR(icol) = - linear_interp(data_beg.data.AGR(icol), data_end.data.AGR(icol), - delta_t_fraction); - data_out.RCO(icol) = - linear_interp(data_beg.data.RCO(icol), data_end.data.RCO(icol), - delta_t_fraction); - data_out.SHP(icol) = - linear_interp(data_beg.data.SHP(icol), data_end.data.SHP(icol), - delta_t_fraction); - data_out.SLV(icol) = - linear_interp(data_beg.data.SLV(icol), data_end.data.SLV(icol), - delta_t_fraction); - data_out.TRA(icol) = - linear_interp(data_beg.data.TRA(icol), data_end.data.TRA(icol), - delta_t_fraction); - data_out.WST(icol) = - linear_interp(data_beg.data.WST(icol), data_end.data.WST(icol), - delta_t_fraction);*/ + for(int i = 0; i < 6; ++i) { + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA( + const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + // We have only 2d vars, so we need to make one team member handle it. + Kokkos::single(Kokkos::PerTeam(team), [&] { + data_out.emiss_components[i](icol) = linear_interp( + data_beg.data.emiss_components[i](icol), + data_end.data.emiss_components[i](icol), delta_t_fraction); + }); }); - }); + } Kokkos::fence(); } // perform_time_interpolation @@ -219,14 +199,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - // Get pointers for the srfEmiss_input - /*auto srfEmiss_data_agr = ekat::scalarize(srfEmiss_input.data.AGR); - auto srfEmiss_data_rco = ekat::scalarize(srfEmiss_input.data.RCO); - auto srfEmiss_data_shp = ekat::scalarize(srfEmiss_input.data.SHP); - auto srfEmiss_data_slv = ekat::scalarize(srfEmiss_input.data.SLV); - auto srfEmiss_data_tra = ekat::scalarize(srfEmiss_input.data.TRA); - auto srfEmiss_data_wst = ekat::scalarize(srfEmiss_input.data.WST); -*/ const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) .get_header() .get_identifier() @@ -239,25 +211,7 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); } - /*auto agr = srfEmiss_horiz_interp.get_tgt_field(0).get_view(); - auto rco = srfEmiss_horiz_interp.get_tgt_field(1).get_view(); - auto shp = srfEmiss_horiz_interp.get_tgt_field(2).get_view(); - auto slv = srfEmiss_horiz_interp.get_tgt_field(3).get_view(); - auto tra = srfEmiss_horiz_interp.get_tgt_field(4).get_view(); - auto wst = srfEmiss_horiz_interp.get_tgt_field(5).get_view(); - - auto copy_and_pad = KOKKOS_LAMBDA(const Member &team) { - int icol = team.league_rank(); - srfEmiss_data_agr(icol) = agr(icol); - srfEmiss_data_rco(icol) = rco(icol); - srfEmiss_data_shp(icol) = shp(icol); - srfEmiss_data_slv(icol) = slv(icol); - srfEmiss_data_tra(icol) = tra(icol); - srfEmiss_data_wst(icol) = wst(icol); - }; - - auto policy = ESU::get_default_team_policy(ncols, 1); - Kokkos::parallel_for("", policy, copy_and_pad);*/ + Kokkos::fence(); stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml index 7363b899a5f..b6e2d545d3c 100644 --- a/components/eamxx/tests/single-process/mam/emissions/output.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -5,56 +5,7 @@ Averaging Type: Instant Fields: Physics: Field Names: - - bc_a1 - - bc_a3 - - bc_a4 - - dst_a1 - - dst_a3 - - so4_a1 - - so4_a2 - - so4_a3 - - pom_a1 - - pom_a3 - - pom_a4 - - soa_a1 - - soa_a2 - - soa_a3 - - nacl_a1 - - nacl_a2 - - nacl_a3 - - mom_a1 - - mom_a2 - - mom_a3 - - mom_a4 - - num_a1 - - num_a2 - - num_a3 - - num_a4 - - bc_c1 - - bc_c3 - - bc_c4 - - dst_c1 - - dst_c3 - - so4_c1 - - so4_c2 - - so4_c3 - - pom_c1 - - pom_c3 - - pom_c4 - - soa_c1 - - soa_c2 - - soa_c3 - - nacl_c1 - - nacl_c2 - - nacl_c3 - - mom_c1 - - mom_c2 - - mom_c3 - - mom_c4 - - num_c1 - - num_c2 - - num_c3 - - num_c4 + - T_mid output_control: Frequency: 1 frequency_units: nsteps From 2d616f77688b2e1754e716c828d9c0572a4e583d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 21:06:09 -0700 Subject: [PATCH 21/65] Removes hardwired num sectors --- ...and_online_emissions_process_interface.cpp | 22 +++++++++------- .../eamxx/src/physics/mam/srf_emission.hpp | 26 ++++++++++++------- .../src/physics/mam/srf_emission_impl.hpp | 22 ++++++++-------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index ff6c0c15605..ffb3cc3f99f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -42,17 +42,19 @@ void MAMSrfOnlineEmiss::set_grids( std::string so2_data_file = "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - std::array so2_fields = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; + static constexpr int num_sectors_so2 = 6; + std::array so2_fields = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; // Init horizontal remap so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( grid_, so2_data_file, so2_fields, srf_map_file); // 2. Initialize the size of the SPAData structures. - srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_); - srfEmissData_out_.init(ncol_, true); // FIXME: should it be true or false??? + srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); + srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); + srfEmissData_out_.init(ncol_, num_sectors_so2, + true); // FIXME: should it be true or false??? // 3. Create reader for srfEmiss data. The reader is an // AtmosphereInput object @@ -96,15 +98,15 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_components[1](i) + std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, srfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_components[2](i) - << ":" << i << std::endl; + std::cout << "BALLI:" << srfEmissData_end_.data.emiss_sectors[2](i) << ":" + << i << std::endl; } } // end initialize_impl() @@ -133,8 +135,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissData_end_, srfEmiss_temp_, srfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_components[2](i) << ":" - << i << std::endl; + std::cout << "BALLI:" << srfEmissData_out_.emiss_sectors[2](i) << ":" << i + << std::endl; } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 37e26bdeb09..c8094d1799d 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -25,29 +25,37 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_) { init(ncol_, true); } + srfEmissData(const int ncol_, const int nsectors_) { + init(ncol_, nsectors_, true); + } - void init(const int ncol_, const bool allocate) { - ncols = ncol_; + void init(const int ncol_, const int nsectors_, const bool allocate) { + ncols = ncol_; + nsectors = nsectors_; if(allocate) { - for(auto &component : emiss_components) { - component = view_1d("", ncols); + for(int i = 0; i < nsectors; ++i) { + auto sector = view_1d("", ncols); + emiss_sectors.push_back(sector); } } } // srfEmissData init // Basic spatial dimensions of the data - int ncols; - std::array emiss_components; + int ncols, nsectors; + std::vector emiss_sectors; }; // srfEmissData struct srfEmissInput { srfEmissInput() = default; - srfEmissInput(const int ncols_) { init(ncols_); } + srfEmissInput(const int ncols_, const int nsectors_) { + init(ncols_, nsectors_); + } - void init(const int ncols_) { data.init(ncols_, true); } + void init(const int ncols_, const int nsectors_) { + data.init(ncols_, nsectors_, true); + } srfEmissData data; // All srfEmiss fields }; // srfEmissInput diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 631862d5b18..a2f6c21bb59 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,14 +57,14 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - std::vector emiss_components; + std::vector emiss_sectors; for(int icomp = 0; icomp < FN; ++icomp) { auto comp_name = field_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); - emiss_components.push_back(f); + emiss_sectors.push_back(f); remapper->register_field_from_tgt(f); } @@ -78,13 +78,13 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector emiss_components; + std::vector emiss_sectors; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - emiss_components.push_back(horiz_remapper->get_src_field(i)); + emiss_sectors.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - emiss_components, true); + emiss_sectors, true); } // create_srfEmiss_data_reader template @@ -127,7 +127,7 @@ void srfEmissFunctions::perform_time_interpolation( const auto policy = ekat::ExeSpaceUtils::get_default_team_policy( data_beg.data.ncols, 1); - for(int i = 0; i < 6; ++i) { + for(int i = 0; i < data_beg.data.nsectors; ++i) { Kokkos::parallel_for( "srfEmiss_time_interp_loop", policy, KOKKOS_LAMBDA( @@ -135,9 +135,9 @@ void srfEmissFunctions::perform_time_interpolation( const int icol = team.league_rank(); // We have only 2d vars, so we need to make one team member handle it. Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_components[i](icol) = linear_interp( - data_beg.data.emiss_components[i](icol), - data_end.data.emiss_components[i](icol), delta_t_fraction); + data_out.emiss_sectors[i](icol) = linear_interp( + data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), delta_t_fraction); }); }); } @@ -207,9 +207,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int ncols = layout.dim(COL); // Read fields from the file - for(int i = 0; i < 6; ++i) { + for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_components[i], aa); + Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], aa); } Kokkos::fence(); From 343bfc3c46683997d5bbd5a53d02039fb12dcf86 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 25 Jul 2024 23:04:33 -0700 Subject: [PATCH 22/65] Creates a function to init srf emiss data structures --- ...and_online_emissions_process_interface.cpp | 86 ++++++++----------- ...and_online_emissions_process_interface.hpp | 22 ++--- .../eamxx/src/physics/mam/srf_emission.hpp | 12 ++- .../src/physics/mam/srf_emission_impl.hpp | 31 ++++++- .../single-process/mam/emissions/input.yaml | 3 +- 5 files changed, 83 insertions(+), 71 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index ffb3cc3f99f..6c8aa494621 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,18 +1,6 @@ #include #include -#include "share/grid/point_grid.hpp" -#include "share/io/scorpio_input.hpp" - -// for SCREAM_CIME_BUILD -#include "scream_config.h" - -/* -Future work: -Write comments -write in/outs for all variables clearly -*/ - namespace scream { // ========================================================================================= @@ -34,33 +22,34 @@ void MAMSrfOnlineEmiss::set_grids( using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); - // Reading so2 srf emiss data - std::string srf_map_file = ""; + // Surface emissions remapping file + std::string srf_map_file = m_params.get("srf_remap_file"); + + //-------------------------------------------------------------------- + // Init so2 srf emiss data structures + //-------------------------------------------------------------------- + // File name std::string so2_data_file = - "/compyfs/inputdata/atm/scream/mam4xx/emissions/test_DECK_ne30/" - "cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc"; - static constexpr int num_sectors_so2 = 6; - std::array so2_fields = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; - - // Init horizontal remap - so2SrfEmissHorizInterp_ = srfEmissFunc::create_horiz_remapper( - grid_, so2_data_file, so2_fields, srf_map_file); - - // 2. Initialize the size of the SPAData structures. - srfEmissData_start_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); - srfEmissData_end_ = srfEmissFunc::srfEmissInput(ncol_, num_sectors_so2); - srfEmissData_out_.init(ncol_, num_sectors_so2, - true); // FIXME: should it be true or false??? - - // 3. Create reader for srfEmiss data. The reader is an - // AtmosphereInput object - srfEmissDataReader_ = srfEmissFunc::create_srfEmiss_data_reader( - so2SrfEmissHorizInterp_, so2_data_file); -} + m_params.get("srf_emis_specifier_for_SO2"); + + // Number of sectors + static constexpr int so2_num_sectors = 6; + + // Sector names in file + std::array so2_sectors = {"AGR", "RCO", "SHP", + "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so2_num_sectors, grid_, so2_data_file, so2_sectors, srf_map_file, + // output + so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, + so2SrfEmissData_out_, so2SrfEmissDataReader_); + +} // set_grid // ========================================================================================= // ON HOST, returns the number of bytes of device memory needed by the above @@ -98,15 +87,15 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. const int curr_month = timestamp().get_month() - 1; // 0-based for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" << srfEmissData_end_.data.emiss_sectors.at(1)(i) - << std::endl; + std::cout << "BALLI-bef:" + << so2SrfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; } srfEmissFunc::update_srfEmiss_data_from_file( - srfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, - srfEmissData_end_); + so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, + so2SrfEmissData_end_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_end_.data.emiss_sectors[2](i) << ":" - << i << std::endl; + std::cout << "BALLI:" << so2SrfEmissData_end_.data.emiss_sectors[2](i) + << ":" << i << std::endl; } } // end initialize_impl() @@ -124,19 +113,18 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { auto ts = timestamp() + dt; // Update the srfEmissTimeState to reflect the current time, note the addition // of dt - srfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - srfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, srfEmissTimeState_, - srfEmissData_start_, srfEmissData_end_); + so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, + so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(srfEmissTimeState_, srfEmissData_start_, - srfEmissData_end_, srfEmiss_temp_, - srfEmissData_out_); + srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, + so2SrfEmissData_end_, so2SrfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srfEmissData_out_.emiss_sectors[2](i) << ":" << i - << std::endl; + std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[2](i) << ":" + << i << std::endl; } /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 202834938e3..c53d22ee32a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -15,15 +15,6 @@ // #include #include -// TODO: determine when these may be necessary -// #ifndef KOKKOS_ENABLE_CUDA -// #define protected_except_cuda public -// #define private_except_cuda public -// #else -// #define protected_except_cuda protected -// #define private_except_cuda private -// #endif - namespace scream { // The process responsible for handling MAM4 surface and online emissions. The @@ -131,16 +122,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { Preprocess preprocess_; // IO structure to read in data for standard grids - std::shared_ptr srfEmissDataReader_; - - srfEmissFunc::srfEmissTimeState srfEmissTimeState_; + std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissInput srfEmissData_start_; - srfEmissFunc::srfEmissInput srfEmissData_end_; - srfEmissFunc::srfEmissOutput srfEmissData_out_; + srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; - // Used to store temporary data during srfEmiss_main - srfEmissFunc::srfEmissInput srfEmiss_temp_; + srfEmissFunc::srfEmissInput so2SrfEmissData_start_; + srfEmissFunc::srfEmissInput so2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; }; // MAMSrfOnlineEmiss diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index c8094d1799d..0add3f343f9 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -80,7 +80,6 @@ struct srfEmissFunctions { static void srfEmiss_main(const srfEmissTimeState &time_state, const srfEmissInput &data_beg, const srfEmissInput &data_end, - const srfEmissInput &data_tmp, // Temporary const srfEmissOutput &data_out); static void update_srfEmiss_data_from_file( @@ -105,6 +104,17 @@ struct srfEmissFunctions { KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, const ScalarX &x1, const ScalarT &t); + template + static void init_srf_emiss_objects( + const int ncol, const int num_sectors, + const std::shared_ptr &grid, + const std::string &data_file, const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader); }; // struct srfEmissFunctions } // namespace diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index a2f6c21bb59..3ca3d85edfc 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -14,7 +14,7 @@ std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &data_file, - const std::array &field_names, + const std::array §or_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -60,7 +60,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector emiss_sectors; for(int icomp = 0; icomp < FN; ++icomp) { - auto comp_name = field_names[icomp]; + auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); @@ -148,7 +148,6 @@ template void srfEmissFunctions::srfEmiss_main(const srfEmissTimeState &time_state, const srfEmissInput &data_beg, const srfEmissInput &data_end, - const srfEmissInput &data_tmp, const srfEmissOutput &data_out) { // Beg/End/Tmp month must have all sizes matching @@ -253,6 +252,32 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate +template +template +void srfEmissFunctions::init_srf_emiss_objects( + const int ncol, const int num_sectors, + const std::shared_ptr &grid, + const std::string &data_file, const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader) { + // Init horizontal remap + SrfEmissHorizInterp = + create_horiz_remapper(grid, data_file, sectors, srf_map_file); + + // Initialize the size of start/end/out data structures + SrfEmissData_start = srfEmissInput(ncol, num_sectors); + SrfEmissData_end = srfEmissInput(ncol, num_sectors); + SrfEmissData_out.init(ncol, num_sectors, true); + + // Create reader (an AtmosphereInput object) + SrfEmissDataReader = + create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); +} // init_srf_emiss_objects + } // namespace } // namespace scream::mam_coupling diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 30861b656e6..69b1a66d975 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -12,8 +12,9 @@ atmosphere_processes: atm_procs_list: [mam4_srf_online_emiss] mam4_srf_online_emiss: # MAM4xx-Surface-Emissions + srf_remap_file: "" srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc From 4ae7b5a1cd2204048681bf32dc22f65c7cab9c0a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 07:07:24 -0700 Subject: [PATCH 23/65] Sums up all emission sectors --- ...and_online_emissions_process_interface.cpp | 2 +- .../src/physics/mam/srf_emission_impl.hpp | 53 ++++++++++++------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 6c8aa494621..490726e7316 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -123,7 +123,7 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[2](i) << ":" + std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 3ca3d85edfc..b459965e69d 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -123,25 +123,40 @@ void srfEmissFunctions::perform_time_interpolation( std::to_string(t_beg) + "\n delta_t: " + std::to_string(delta_t) + "\n"); - using KT = ekat::KokkosTypes; - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy( - data_beg.data.ncols, 1); - for(int i = 0; i < data_beg.data.nsectors; ++i) { - Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA( - const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - // We have only 2d vars, so we need to make one team member handle it. - Kokkos::single(Kokkos::PerTeam(team), [&] { - data_out.emiss_sectors[i](icol) = linear_interp( - data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), delta_t_fraction); - }); - }); + for(int icol = 0; icol < data_beg.data.ncols; ++icol) { + Real accum = 0; + for(int i = 0; i < data_beg.data.nsectors; ++i) { + accum += + linear_interp(data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), delta_t_fraction); + } + data_out.emiss_sectors[0](icol) = accum; } - Kokkos::fence(); + + /*const auto policy = ESU::get_default_team_policy(data_beg.data.ncols, 1); + + Kokkos::parallel_for( + "srfEmiss_time_interp_loop", policy, + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); + double result; + + Kokkos::parallel_reduce("Loop1", N, KOKKOS_LAMBDA (const int& i, double& + lsum) { lsum += 1.0*i; + }, result); + + Kokkos::parallel_reduce("srfEmiss_reduction_loop", N, KOKKOS_LAMBDA + (const int& i, double& lsum) { for(int i = 0; i < data_beg.data.nsectors; ++i) + { Kokkos::single(Kokkos::PerTeam(team), [&] { accum[icol] = accum[icol] + + linear_interp(data_beg.data.emiss_sectors[i](icol), + data_end.data.emiss_sectors[i](icol), + delta_t_fraction); + + }); + } + data_out.emiss_sectors[0](icol) = accum[icol]; + }); + Kokkos::fence();*/ } // perform_time_interpolation template @@ -271,7 +286,7 @@ void srfEmissFunctions::init_srf_emiss_objects( // Initialize the size of start/end/out data structures SrfEmissData_start = srfEmissInput(ncol, num_sectors); SrfEmissData_end = srfEmissInput(ncol, num_sectors); - SrfEmissData_out.init(ncol, num_sectors, true); + SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) SrfEmissDataReader = From 9769a95a841f4c2f8e84805c2c008351f8fccc11 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 10:11:57 -0700 Subject: [PATCH 24/65] Adds bc_a4 and dms emmisions --- .../cime_config/namelist_defaults_scream.xml | 32 ++--- ...and_online_emissions_process_interface.cpp | 119 +++++++++++++++--- ...and_online_emissions_process_interface.hpp | 23 ++-- .../src/physics/mam/srf_emission_impl.hpp | 1 + .../single-process/mam/emissions/input.yaml | 29 ++--- 5 files changed, 140 insertions(+), 64 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index bb36bdc2ec7..e8e062cfaeb 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,28 +269,18 @@ be lost if SCREAM_HACK_XML is not enabled. - - - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc + - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc - + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 490726e7316..14bd3621af6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -29,6 +29,25 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); + //-------------------------------------------------------------------- + // Init dms srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string dms_data_file = + m_params.get("srf_emis_specifier_for_DMS"); + + // Number of sectors + static constexpr int dms_num_sectors = 1; + + // Sector names in file + std::array dms_sectors = {"DMS"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, dms_num_sectors, grid_, dms_data_file, dms_sectors, srf_map_file, + // output + dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, + dmsSrfEmissData_out_, dmsSrfEmissDataReader_); + //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- @@ -49,6 +68,27 @@ void MAMSrfOnlineEmiss::set_grids( so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_, so2SrfEmissDataReader_); + //-------------------------------------------------------------------- + // Init bc_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string bc_a4_data_file = + m_params.get("srf_emis_specifier_for_bc_a4"); + + // Number of sectors + static constexpr int bc_a4_num_sectors = 8; + + // Sector names in file + std::array bc_a4_sectors = { + "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, bc_a4_num_sectors, grid_, bc_a4_data_file, bc_a4_sectors, + srf_map_file, + // output + bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + } // set_grid // ========================================================================================= @@ -73,28 +113,39 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { used_mem == requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= -// // TODO: comments! -// void MAMSrfOnlineEmiss::set_emissions_names() {} \\ end set_emissions_names() -// ========================================================================================= -// inline void set_emissions_layouts() { -// } // end set_emissions_layouts() // ========================================================================================= void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + const int curr_month = timestamp().get_month() - 1; // 0-based + // Load the first month into srfEmiss_end. + // Note: At the first time step, the data will be moved into srfEmiss_beg, - // and srfEmiss_end will be reloaded from file with the new month. - const int curr_month = timestamp().get_month() - 1; // 0-based - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI-bef:" - << so2SrfEmissData_end_.data.emiss_sectors.at(1)(i) << std::endl; - } + // and srfEmiss_end will be reloaded from file with the new month. + + //-------------------------------------------------------------------- + // Update dms srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + dmsSrfEmissDataReader_, timestamp(), curr_month, *dmsSrfEmissHorizInterp_, + dmsSrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so2 srf emiss from file + //-------------------------------------------------------------------- srfEmissFunc::update_srfEmiss_data_from_file( so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, so2SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update bc_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + bc_a4SrfEmissDataReader_, timestamp(), curr_month, + *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_end_.data.emiss_sectors[2](i) + std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) << ":" << i << std::endl; } @@ -111,9 +162,28 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Gather time and state information for interpolation auto ts = timestamp() + dt; - // Update the srfEmissTimeState to reflect the current time, note the addition - // of dt + + //-------------------------------------------------------------------- + // Interpolate dms srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + dmsSrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + dmsSrfEmissDataReader_, ts, *dmsSrfEmissHorizInterp_, + dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, + dmsSrfEmissData_end_, dmsSrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, @@ -122,8 +192,25 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate bc_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, + bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so2SrfEmissData_out_.emiss_sectors[0](i) << ":" + std::cout << "BALLI:" << dmsSrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index c53d22ee32a..b5390c44e52 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -38,9 +38,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; - // Structures to store the data used for interpolation - std::shared_ptr so2SrfEmissHorizInterp_; - public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -122,14 +119,26 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { Preprocess preprocess_; // IO structure to read in data for standard grids - std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; + std::shared_ptr dmsSrfEmissHorizInterp_; + std::shared_ptr dmsSrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; + srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; + srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - srfEmissFunc::srfEmissInput so2SrfEmissData_start_; - srfEmissFunc::srfEmissInput so2SrfEmissData_end_; + // Structures to store the data used for interpolation + std::shared_ptr so2SrfEmissHorizInterp_; + std::shared_ptr so2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; + std::shared_ptr bc_a4SrfEmissHorizInterp_; + std::shared_ptr bc_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b459965e69d..b924706ed4a 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -129,6 +129,7 @@ void srfEmissFunctions::perform_time_interpolation( accum += linear_interp(data_beg.data.emiss_sectors[i](icol), data_end.data.emiss_sectors[i](icol), delta_t_fraction); + if(icol == 19) std::cout << "accum:" << accum << std::endl; } data_out.emiss_sectors[0](icol) = accum; } diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 69b1a66d975..2e63314ebb1 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,26 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc - # MAM4xx-Online-Emissions - online_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_SOAG: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_soag_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_elev_1x1_2010_clim_c20190821.nc - online_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_elev_1x1_2010_clim_c20190821.nc - + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From 14e944ffa6020a147ba392445af9af9869700c41 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 26 Jul 2024 16:17:02 -0700 Subject: [PATCH 25/65] Adds all the emission files --- ...and_online_emissions_process_interface.cpp | 279 +++++++++++++++++- ...and_online_emissions_process_interface.hpp | 42 +++ 2 files changed, 320 insertions(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 14bd3621af6..7899020ce39 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -89,6 +89,140 @@ void MAMSrfOnlineEmiss::set_grids( bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + //-------------------------------------------------------------------- + // Init num_a1 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a1_data_file = + m_params.get("srf_emis_specifier_for_num_a1"); + + // Number of sectors + static constexpr int num_a1_num_sectors = 4; + + // Sector names in file + std::array num_a1_sectors = { + "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a1_num_sectors, grid_, num_a1_data_file, num_a1_sectors, + srf_map_file, + // output + num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, + num_a1SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init num_a2 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a2_data_file = + m_params.get("srf_emis_specifier_for_num_a2"); + + // Number of sectors + static constexpr int num_a2_num_sectors = 2; + + // Sector names in file + std::array num_a2_sectors = { + "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a2_num_sectors, grid_, num_a2_data_file, num_a2_sectors, + srf_map_file, + // output + num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, + num_a2SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init num_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string num_a4_data_file = + m_params.get("srf_emis_specifier_for_num_a4"); + + // Number of sectors + static constexpr int num_a4_num_sectors = 16; + + // Sector names in file + std::array num_a4_sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, num_a4_num_sectors, grid_, num_a4_data_file, num_a4_sectors, + srf_map_file, + // output + num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, + num_a4SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init pom_a4 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string pom_a4_data_file = + m_params.get("srf_emis_specifier_for_pom_a4"); + + // Number of sectors + static constexpr int pom_a4_num_sectors = 8; + + // Sector names in file + std::array pom_a4_sectors = { + "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, pom_a4_num_sectors, grid_, pom_a4_data_file, pom_a4_sectors, + srf_map_file, + // output + pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, + pom_a4SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init so4_a1 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string so4_a1_data_file = + m_params.get("srf_emis_specifier_for_so4_a1"); + + // Number of sectors + static constexpr int so4_a1_num_sectors = 4; + + // Sector names in file + std::array so4_a1_sectors = {"AGR", "SHP", + "SLV", "WST"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so4_a1_num_sectors, grid_, so4_a1_data_file, so4_a1_sectors, + srf_map_file, + // output + so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, + so4_a1SrfEmissDataReader_); + + //-------------------------------------------------------------------- + // Init so4_a2 srf emiss data structures + //-------------------------------------------------------------------- + // File name + std::string so4_a2_data_file = + m_params.get("srf_emis_specifier_for_so4_a2"); + + // Number of sectors + static constexpr int so4_a2_num_sectors = 2; + + // Sector names in file + std::array so4_a2_sectors = {"RCO", "TRA"}; + + srfEmissFunc::init_srf_emiss_objects( + ncol_, so4_a2_num_sectors, grid_, so4_a2_data_file, so4_a2_sectors, + srf_map_file, + // output + so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, + so4_a2SrfEmissDataReader_); + } // set_grid // ========================================================================================= @@ -144,6 +278,47 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { bc_a4SrfEmissDataReader_, timestamp(), curr_month, *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); + //-------------------------------------------------------------------- + // Update num_a1 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a1SrfEmissDataReader_, timestamp(), curr_month, + *num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update num_a2 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a2SrfEmissDataReader_, timestamp(), curr_month, + *num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update num_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + num_a4SrfEmissDataReader_, timestamp(), curr_month, + *num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update pom_a4 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + pom_a4SrfEmissDataReader_, timestamp(), curr_month, + *pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so4_a1 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + so4_a1SrfEmissDataReader_, timestamp(), curr_month, + *so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_end_); + + //-------------------------------------------------------------------- + // Update so4_a2 srf emiss from file + //-------------------------------------------------------------------- + srfEmissFunc::update_srfEmiss_data_from_file( + so4_a2SrfEmissDataReader_, timestamp(), curr_month, + *so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_end_); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) << ":" << i << std::endl; @@ -209,8 +384,110 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); + //-------------------------------------------------------------------- + // Interpolate num_a1 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, + num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, + num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate num_a2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, + num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, + num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate num_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, + num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, + num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate pom_a4 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, + pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, + pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so4_a1 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, + so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, + so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); + + //-------------------------------------------------------------------- + // Interpolate so4_a2 srf emiss data + //-------------------------------------------------------------------- + // Update srfEmissTimeState, note the addition of dt + so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, + so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, + so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); + for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << dmsSrfEmissData_out_.emiss_sectors[0](i) << ":" + std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" << i << std::endl; } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index b5390c44e52..dc0ab7e13bc 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -139,6 +139,48 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + std::shared_ptr num_a1SrfEmissHorizInterp_; + std::shared_ptr num_a1SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; + + std::shared_ptr num_a2SrfEmissHorizInterp_; + std::shared_ptr num_a2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; + + std::shared_ptr num_a4SrfEmissHorizInterp_; + std::shared_ptr num_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; + + std::shared_ptr pom_a4SrfEmissHorizInterp_; + std::shared_ptr pom_a4SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; + srfEmissFunc::srfEmissInput pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_; + srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; + + std::shared_ptr so4_a1SrfEmissHorizInterp_; + std::shared_ptr so4_a1SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; + + std::shared_ptr so4_a2SrfEmissHorizInterp_; + std::shared_ptr so4_a2SrfEmissDataReader_; + srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; + srfEmissFunc::srfEmissInput so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_; + srfEmissFunc::srfEmissOutput so4_a2SrfEmissData_out_; + }; // MAMSrfOnlineEmiss } // namespace scream From b073225b912008ec12bd306fdd39dfa72182bd8f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 06:32:15 -0700 Subject: [PATCH 26/65] Computes emissions flux and adds use of molecular weights for unit conversion --- ...and_online_emissions_process_interface.cpp | 105 ++++++++++++++++-- ...and_online_emissions_process_interface.hpp | 68 +++++++----- .../src/physics/mam/mam_emissions_utils.hpp | 91 --------------- .../src/physics/mam/srf_emission_impl.hpp | 14 +++ 4 files changed, 150 insertions(+), 128 deletions(-) delete mode 100644 components/eamxx/src/physics/mam/mam_emissions_utils.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 7899020ce39..21bf8a5bc1b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -3,7 +3,9 @@ namespace scream { -// ========================================================================================= +// ================================================================ +// Constructor +// ================================================================ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { @@ -12,7 +14,9 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, */ } -// ========================================================================================= +// ================================================================ +// SET_GRIDS +// ================================================================ void MAMSrfOnlineEmiss::set_grids( const std::shared_ptr grids_manager) { grid_ = grids_manager->get_grid("Physics"); @@ -23,9 +27,24 @@ void MAMSrfOnlineEmiss::set_grids( using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + static constexpr int pcnst = mam4::aero_model::pcnst; + + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- + static constexpr Units m2(m * m, "m2"); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + grid_name); + // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); @@ -225,14 +244,18 @@ void MAMSrfOnlineEmiss::set_grids( } // set_grid -// ========================================================================================= +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ // ON HOST, returns the number of bytes of device memory needed by the above // Buffer type given the number of columns and vertical levels size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { return mam_coupling::buffer_size(ncol_, nlev_); } -// ========================================================================================= +// ================================================================ +// INIT_BUFFERS +// ================================================================ // ON HOST, initializes the Buffer type with sufficient memory to store // intermediate (dry) quantities on the given number of columns with the given // number of vertical levels. Returns the number of bytes allocated. @@ -248,9 +271,17 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); } -// ========================================================================================= +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { - const int curr_month = timestamp().get_month() - 1; // 0-based + // --------------------------------------------------------------- + // Output fields + // --------------------------------------------------------------- + constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); + + // Current month ( 0-based) + const int curr_month = timestamp().get_month() - 1; // Load the first month into srfEmiss_end. @@ -324,16 +355,23 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { << ":" << i << std::endl; } + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + preprocess_.initialize(constituent_fluxes_); + } // end initialize_impl() -// ============================================================================= +// ================================================================ +// RUN_IMPL +// ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); // preprocess input -- needs a scan for the calculation of atm height - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -353,6 +391,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, dmsSrfEmissData_out_); + // update flux + auto constituent_fluxes_DMS = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(constituent_fluxes_DMS, + dmsSrfEmissData_out_.emiss_sectors[0]); + //-------------------------------------------------------------------- // Interpolate so2 srf emiss data //-------------------------------------------------------------------- @@ -367,6 +411,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); + // update flux + auto constituent_fluxes_SO2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::SO2)); + Kokkos::deep_copy(constituent_fluxes_SO2, + so2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate bc_a4 srf emiss data @@ -383,6 +432,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_bc_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::bc_a4)); + Kokkos::deep_copy(constituent_fluxes_bc_a4, + bc_a4SrfEmissData_out_.emiss_sectors[0]); + //-------------------------------------------------------------------- // Interpolate num_a1 srf emiss data @@ -400,6 +455,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a1 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a1)); + Kokkos::deep_copy(constituent_fluxes_num_a1, + num_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a2 srf emiss data @@ -417,6 +477,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a2)); + Kokkos::deep_copy(constituent_fluxes_num_a2, + num_a2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a4 srf emiss data @@ -434,6 +499,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_num_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a4)); + Kokkos::deep_copy(constituent_fluxes_num_a4, + num_a4SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate pom_a4 srf emiss data @@ -451,6 +521,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); +// update flux + auto constituent_fluxes_pom_a4 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::pom_a4)); + Kokkos::deep_copy(constituent_fluxes_pom_a4, + pom_a4SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate so4_a1 srf emiss data @@ -468,6 +543,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); +// update flux + auto constituent_fluxes_so4_a1 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a1)); + Kokkos::deep_copy(constituent_fluxes_so4_a1, + so4_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate so4_a2 srf emiss data @@ -485,10 +565,15 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); +// update flux + auto constituent_fluxes_so4_a2 = Kokkos::subview( + constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a2)); + Kokkos::deep_copy(constituent_fluxes_so4_a2, + so4_a2SrfEmissData_out_.emiss_sectors[0]); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << std::endl; + << i <<":"< #include #include -#include +#include "share/grid/remap/abstract_remapper.hpp" +#include "share/io/scorpio_input.hpp" // For MAM4 aerosol configuration #include #include @@ -20,7 +21,8 @@ namespace scream { // The process responsible for handling MAM4 surface and online emissions. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; + using KT = ekat::KokkosTypes; + using view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -38,6 +40,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // physics grid for column information std::shared_ptr grid_; + view_2d constituent_fluxes_; + public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -79,66 +83,71 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct Preprocess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data - void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere &wet_atm, - const mam_coupling::AerosolState &wet_aero, - const mam_coupling::DryAtmosphere &dry_atm, - const mam_coupling::AerosolState &dry_aero) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; - wet_aero_pre_ = wet_aero; - dry_atm_pre_ = dry_atm; - dry_aero_pre_ = dry_aero; + void initialize(const view_2d &constituent_fluxes) { + constituent_fluxes_pre_ = constituent_fluxes; } KOKKOS_INLINE_FUNCTION void operator()( const Kokkos::TeamPolicy::member_type &team) const { const int i = team.league_rank(); // column index - - compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, - dry_aero_pre_, i); + // zero-out the constituent surface fluxes for all gas and aerosol + // species. + for(auto icnst = mam4::utils::gasses_start_ind(); + icnst < mam4::aero_model::pcnst; ++icnst) { + constituent_fluxes_pre_(i, icnst) = 0; + } team.team_barrier(); } // operator() // local variables for preprocess struct - // number of horizontal columns and vertical levels - int ncol_pre_, nlev_pre_; + view_2d constituent_fluxes_pre_; - // local atmospheric and aerosol state data - mam_coupling::WetAtmosphere wet_atm_pre_; - mam_coupling::DryAtmosphere dry_atm_pre_; - mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; - }; // MAMAci::Preprocess + }; // MAMSrfOnlineEmiss::Preprocess private: // preprocessing scratch pad Preprocess preprocess_; - // IO structure to read in data for standard grids - + // Species index in tracer array with "pcnst" indices + enum class spcIndex_in_pcnst : int { + SO2 = 12, + DMS = 13, + so4_a1 = 15, + num_a1 = 22, + so4_a2 = 23, + num_a2 = 27, + pom_a4 = 36, + bc_a4 = 37, + num_a4 = 39, + }; + + //offset for converting pcnst index to gas_pcnst index + static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; + + // Data structures to read DMS data file std::shared_ptr dmsSrfEmissHorizInterp_; std::shared_ptr dmsSrfEmissDataReader_; srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - // Structures to store the data used for interpolation + // Data structures to read so2 data file std::shared_ptr so2SrfEmissHorizInterp_; std::shared_ptr so2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; + // Data structures to read bc_a4 data file std::shared_ptr bc_a4SrfEmissHorizInterp_; std::shared_ptr bc_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; + // Data structures to read num_a1 data file std::shared_ptr num_a1SrfEmissHorizInterp_; std::shared_ptr num_a1SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; @@ -146,6 +155,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a1SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; + // Data structures to read num_a2 data file std::shared_ptr num_a2SrfEmissHorizInterp_; std::shared_ptr num_a2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; @@ -153,6 +163,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a2SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; + // Data structures to read num_a4 data file std::shared_ptr num_a4SrfEmissHorizInterp_; std::shared_ptr num_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; @@ -160,6 +171,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; + // Data structures to read pom_a4 data file std::shared_ptr pom_a4SrfEmissHorizInterp_; std::shared_ptr pom_a4SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; @@ -167,6 +179,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { pom_a4SrfEmissData_end_; srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; + // Data structures to read so4_a1 data file std::shared_ptr so4_a1SrfEmissHorizInterp_; std::shared_ptr so4_a1SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; @@ -174,6 +187,7 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { so4_a1SrfEmissData_end_; srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; + // Data structures to read so4_a2 data file std::shared_ptr so4_a2SrfEmissHorizInterp_; std::shared_ptr so4_a2SrfEmissDataReader_; srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; diff --git a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp b/components/eamxx/src/physics/mam/mam_emissions_utils.hpp deleted file mode 100644 index 527efd5de18..00000000000 --- a/components/eamxx/src/physics/mam/mam_emissions_utils.hpp +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef MAM_EMISSIONS_READ_TABLES_HPP -#define MAM_EMISSIONS_READ_TABLES_HPP - -#include "ekat/ekat_parameter_list.hpp" -#include "mam_coupling.hpp" -#include "share/field/field_manager.hpp" -#include "share/grid/abstract_grid.hpp" -#include "share/grid/grids_manager.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_scorpio_interface.hpp" - -// later to mam_coupling.hpp -namespace scream::mam_coupling { - -using view_1d_host = typename KT::view_1d::HostMirror; -using view_1d_int_host = typename KT::view_1d::HostMirror; -using view_2d_host = typename KT::view_2d::HostMirror; -// using view_5d_host = typename KT::view_ND::HostMirror; -// using complex_view_1d = typename KT::view_1d>; - -constexpr int n_srf_emiss = mam4::mo_srf_emissions::n_srf_emiss; -constexpr int n_online_emiss = mam4::aero_model_emissions::n_online_emiss; - -// FIXME: this will need to change when we remap to flattened column idx -constexpr int nlat_srf = 96; -constexpr int nlon_srf = 144; - -constexpr int nalti_online = 13; -constexpr int nlat_online = 96; -constexpr int nlon_online = 144; - -using namespace ShortFieldTagsNames; - -// std::map> map_srf_emiss_file_vars; - -inline void set_file_var_names(std::map> &var_map, - std::map &spec_map) { - // for (const auto &spec : spec_map) { - // std::string spec_name = spec.first; - // std::cout << "var_map[spec_name] = " << var_map[spec_name] << "\n"; - // } -} - -// struct AerosolSurfaceEmissionsHostData { -// // these have dim = n_species -// view_1d_host emis_species_index; -// view_1d_host emis_species_units; -// view_1d_host emis_species_name; -// // molecular weight -// view_1d_host emis_species_mw; -// // number of sectors in each field -// view_1d_int_host emis_species_nsectors; -// // FIXME: not quite sure what this does--maybe just a placeholder for -// // fields(:, i_sector)? -// view_1d_host emis_species_sector; -// // note fields have dim = n_species x nsectors -// // TODO: fields have units??? maybe the same as the upper spec units -// view_2d_host emis_species_fields; -// }; - -// using AerosolSurfaceEmissionsDeviceData = - // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDeviceData; - -// inline void set_emissions_params( -// AerosolSurfaceEmissionsHostData& aerosol_emissions_host_data, -// ekat::ParameterList& params_emissions, -// std::map& layouts, -// std::map& host_views) { -// // Set up input structure to read data from file. -// using strvec_t = std::vector; -// // using namespace ShortFieldTagsNames; - -// // using SrfEmisDims = -// // mam4::mo_srf_emissions::AerosolSurfaceEmissionsDimensions; SrfEmisDims -// // srf_emimssions_dims; -// } - -// inline void set_emissions_names(const std::map map_spec_id, -// const std::string emis_type, -// const ekat::ParameterList& m_params, -// std::map& host_views) { - -// using view_1d_host = typename KT::view_1d::HostMirror; - -// std::string - -// } // end set_emissions_names - -} // namespace scream::mam_coupling - -#endif diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b924706ed4a..0934ad5fe7b 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -4,6 +4,20 @@ #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scorpio_input.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" +#include "share/scream_types.hpp" + +#include +#include +#include +#include +#include namespace scream::mam_coupling { namespace { From 89ec5e5ebfb178d2b83952259f491a489f1dceb6 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 07:16:19 -0700 Subject: [PATCH 27/65] Removes num_sectors from the arg list --- ...and_online_emissions_process_interface.cpp | 92 ++++++++++--------- ...and_online_emissions_process_interface.hpp | 6 +- .../eamxx/src/physics/mam/srf_emission.hpp | 3 +- .../src/physics/mam/srf_emission_impl.hpp | 26 +++--- 4 files changed, 63 insertions(+), 64 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 21bf8a5bc1b..5f396ef4df4 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -62,7 +62,7 @@ void MAMSrfOnlineEmiss::set_grids( std::array dms_sectors = {"DMS"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, dms_num_sectors, grid_, dms_data_file, dms_sectors, srf_map_file, + ncol_, grid_, dms_data_file, dms_sectors, srf_map_file, // output dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, dmsSrfEmissData_out_, dmsSrfEmissDataReader_); @@ -82,7 +82,7 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so2_num_sectors, grid_, so2_data_file, so2_sectors, srf_map_file, + ncol_, grid_, so2_data_file, so2_sectors, srf_map_file, // output so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_, so2SrfEmissDataReader_); @@ -102,8 +102,7 @@ void MAMSrfOnlineEmiss::set_grids( "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, bc_a4_num_sectors, grid_, bc_a4_data_file, bc_a4_sectors, - srf_map_file, + ncol_, grid_, bc_a4_data_file, bc_a4_sectors, srf_map_file, // output bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); @@ -123,8 +122,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a1_num_sectors, grid_, num_a1_data_file, num_a1_sectors, - srf_map_file, + ncol_, grid_, num_a1_data_file, num_a1_sectors, srf_map_file, // output num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, @@ -145,8 +143,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a2_num_sectors, grid_, num_a2_data_file, num_a2_sectors, - srf_map_file, + ncol_, grid_, num_a2_data_file, num_a2_sectors, srf_map_file, // output num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, @@ -170,8 +167,7 @@ void MAMSrfOnlineEmiss::set_grids( "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, num_a4_num_sectors, grid_, num_a4_data_file, num_a4_sectors, - srf_map_file, + ncol_, grid_, num_a4_data_file, num_a4_sectors, srf_map_file, // output num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, @@ -192,8 +188,7 @@ void MAMSrfOnlineEmiss::set_grids( "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, pom_a4_num_sectors, grid_, pom_a4_data_file, pom_a4_sectors, - srf_map_file, + ncol_, grid_, pom_a4_data_file, pom_a4_sectors, srf_map_file, // output pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, @@ -214,8 +209,7 @@ void MAMSrfOnlineEmiss::set_grids( "SLV", "WST"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so4_a1_num_sectors, grid_, so4_a1_data_file, so4_a1_sectors, - srf_map_file, + ncol_, grid_, so4_a1_data_file, so4_a1_sectors, srf_map_file, // output so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, @@ -235,8 +229,7 @@ void MAMSrfOnlineEmiss::set_grids( std::array so4_a2_sectors = {"RCO", "TRA"}; srfEmissFunc::init_srf_emiss_objects( - ncol_, so4_a2_num_sectors, grid_, so4_a2_data_file, so4_a2_sectors, - srf_map_file, + ncol_, grid_, so4_a2_data_file, so4_a2_sectors, srf_map_file, // output so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, @@ -392,8 +385,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { dmsSrfEmissData_end_, dmsSrfEmissData_out_); // update flux - auto constituent_fluxes_DMS = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::DMS)); + auto constituent_fluxes_DMS = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); Kokkos::deep_copy(constituent_fluxes_DMS, dmsSrfEmissData_out_.emiss_sectors[0]); @@ -412,8 +406,9 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_, so2SrfEmissData_out_); // update flux - auto constituent_fluxes_SO2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::SO2)); + auto constituent_fluxes_SO2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::SO2)); Kokkos::deep_copy(constituent_fluxes_SO2, so2SrfEmissData_out_.emiss_sectors[0]); @@ -432,13 +427,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // Call the main srfEmiss routine to get interpolated aerosol forcings. srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_bc_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::bc_a4)); + // update flux + auto constituent_fluxes_bc_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::bc_a4)); Kokkos::deep_copy(constituent_fluxes_bc_a4, bc_a4SrfEmissData_out_.emiss_sectors[0]); - //-------------------------------------------------------------------- // Interpolate num_a1 srf emiss data //-------------------------------------------------------------------- @@ -455,11 +450,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a1 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a1)); + // update flux + auto constituent_fluxes_num_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a1)); Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); + num_a1SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a2 srf emiss data @@ -477,11 +473,12 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a2)); + // update flux + auto constituent_fluxes_num_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a2)); Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); + num_a2SrfEmissData_out_.emiss_sectors[0]); //-------------------------------------------------------------------- // Interpolate num_a4 srf emiss data @@ -499,9 +496,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_num_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::num_a4)); + // update flux + auto constituent_fluxes_num_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a4)); Kokkos::deep_copy(constituent_fluxes_num_a4, num_a4SrfEmissData_out_.emiss_sectors[0]); @@ -521,9 +519,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); -// update flux - auto constituent_fluxes_pom_a4 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::pom_a4)); + // update flux + auto constituent_fluxes_pom_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::pom_a4)); Kokkos::deep_copy(constituent_fluxes_pom_a4, pom_a4SrfEmissData_out_.emiss_sectors[0]); @@ -543,9 +542,10 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); -// update flux - auto constituent_fluxes_so4_a1 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a1)); + // update flux + auto constituent_fluxes_so4_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a1)); Kokkos::deep_copy(constituent_fluxes_so4_a1, so4_a1SrfEmissData_out_.emiss_sectors[0]); @@ -565,15 +565,17 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); -// update flux - auto constituent_fluxes_so4_a2 = Kokkos::subview( - constituent_fluxes_, Kokkos::ALL(), static_cast(spcIndex_in_pcnst::so4_a2)); + // update flux + auto constituent_fluxes_so4_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a2)); Kokkos::deep_copy(constituent_fluxes_so4_a2, so4_a2SrfEmissData_out_.emiss_sectors[0]); for(int i = 19; i < 30; ++i) { std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i <<":"< #include #include + #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" // For MAM4 aerosol configuration @@ -123,8 +124,9 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4 = 39, }; - //offset for converting pcnst index to gas_pcnst index - static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; + // offset for converting pcnst index to gas_pcnst index + static constexpr int offset_ = + mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; // Data structures to read DMS data file std::shared_ptr dmsSrfEmissHorizInterp_; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 0add3f343f9..b1d45b2855e 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -106,8 +106,7 @@ struct srfEmissFunctions { const ScalarT &t); template static void init_srf_emiss_objects( - const int ncol, const int num_sectors, - const std::shared_ptr &grid, + const int ncol, const std::shared_ptr &grid, const std::string &data_file, const std::array §ors, const std::string &srf_map_file, // output diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 0934ad5fe7b..4dc438b5a9a 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -1,23 +1,20 @@ #ifndef SRF_EMISSION_IMPL_HPP #define SRF_EMISSION_IMPL_HPP +#include +#include +#include +#include +#include + +#include "physics/share/physics_constants.hpp" #include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" #include "share/io/scorpio_input.hpp" -#include "physics/share/physics_constants.hpp" -#include "share/grid/remap/coarsening_remapper.hpp" -#include "share/grid/remap/refining_remapper_p2p.hpp" -#include "share/grid/remap/identity_remapper.hpp" #include "share/io/scream_scorpio_interface.hpp" -#include "share/util/scream_timing.hpp" #include "share/scream_types.hpp" - -#include -#include -#include -#include -#include +#include "share/util/scream_timing.hpp" namespace scream::mam_coupling { namespace { @@ -285,8 +282,7 @@ void srfEmissFunctions::update_srfEmiss_timestate( template template void srfEmissFunctions::init_srf_emiss_objects( - const int ncol, const int num_sectors, - const std::shared_ptr &grid, + const int ncol, const std::shared_ptr &grid, const std::string &data_file, const std::array §ors, const std::string &srf_map_file, // output @@ -299,8 +295,8 @@ void srfEmissFunctions::init_srf_emiss_objects( create_horiz_remapper(grid, data_file, sectors, srf_map_file); // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, num_sectors); - SrfEmissData_end = srfEmissInput(ncol, num_sectors); + SrfEmissData_start = srfEmissInput(ncol, FN); + SrfEmissData_end = srfEmissInput(ncol, FN); SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) From 5db0b8a7033ce8ca178cd925cba92707b7f0366c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 16:34:13 -0700 Subject: [PATCH 28/65] Partial implementation of a vector for surface emissions --- ...and_online_emissions_process_interface.cpp | 35 ++++++ ...and_online_emissions_process_interface.hpp | 24 +++++ .../eamxx/src/physics/mam/srf_emission.hpp | 24 ++++- .../src/physics/mam/srf_emission_impl.hpp | 102 ++++++++++++++++-- 4 files changed, 174 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 5f396ef4df4..d855a292926 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -48,6 +48,41 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); + // EXPERIMENTAL + /*using srf_emission_variant = + std::variant, srf_emiss<2>, srf_emiss<4>, srf_emiss<6>, + srf_emiss<8>, srf_emiss<16>>; + + std::vector srfballivec; + + static constexpr int dms_num_sectors1 = 1; + srf_emiss dms1; + dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms1.sectors = {"DMS"}; + + srfballivec.push_back(dms1); + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, srfballivec[0].data_file , srfballivec[0].sectors, + srf_map_file, + // output + srfballivec[0].HorizInterp_, srfballivec[0].EmissData_start_, + srfballivec[0].EmissData_end_, srfballivec[0].EmissData_out_, + srfballivec[0].EmissDataReader_);*/ + + std::vector srfballivec; + srf_emiss dms1; + dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms1.sectors = {"DMS"}; + srfballivec.push_back(dms1); + + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, srfballivec[0].data_file, srfballivec[0].sectors, + srf_map_file, + // output + srfballivec[0].HorizInterp_, srfballivec[0].Data_start_, + srfballivec[0].Data_end_, srfballivec[0].Data_out_, + srfballivec[0].DataReader_); + //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 18c3712ed5f..dca30a176dd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -16,6 +16,7 @@ #include // #include #include +#include namespace scream { @@ -124,6 +125,29 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { num_a4 = 39, }; + // A struct carrying all the fields needed to read + // surface emissions of a species + // template + struct srf_emiss { + // srf_emiss(int n){numSectors=n;} + // data file name + std::string data_file; + // static int numSectors; + // Sector names in file + std::vector sectors; + + std::shared_ptr HorizInterp_; + std::shared_ptr DataReader_; + srfEmissFunc::srfEmissTimeState TimeState_; + srfEmissFunc::srfEmissInput Data_start_, Data_end_; + srfEmissFunc::srfEmissOutput Data_out_; + + /*srfEmissFunc::init_srf_emiss_objects( + // output + dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, + dmsSrfEmissData_out_, dmsSrfEmissDataReader_);*/ + }; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index b1d45b2855e..585656b0ec1 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -66,13 +66,18 @@ struct srfEmissFunctions { /* ------------------------------------------------------------------------------------------- */ // Surface emissions routines - template + template static std::shared_ptr create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &srfEmiss_data_file, - const std::array &field_names, + const std::array &field_names, const std::string &map_file); + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &srfEmiss_data_file, + const std::vector &field_names, const std::string &map_file); + static std::shared_ptr create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file); @@ -104,10 +109,21 @@ struct srfEmissFunctions { KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, const ScalarX &x1, const ScalarT &t); - template + template + static void init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, + const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader); + static void init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, - const std::string &data_file, const std::array §ors, + const std::string &data_file, const std::vector §ors, const std::string &srf_map_file, // output std::shared_ptr &SrfEmissHorizInterp, diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 4dc438b5a9a..b792c805b32 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -20,12 +20,12 @@ namespace scream::mam_coupling { namespace { template -template +template std::shared_ptr srfEmissFunctions::create_horiz_remapper( const std::shared_ptr &model_grid, const std::string &data_file, - const std::array §or_names, + const std::array §or_names, const std::string &map_file) { using namespace ShortFieldTagsNames; @@ -70,7 +70,70 @@ srfEmissFunctions::create_horiz_remapper( std::vector emiss_sectors; - for(int icomp = 0; icomp < FN; ++icomp) { + for(int icomp = 0; icomp < numSectors; ++icomp) { + auto comp_name = sector_names[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + f.allocate_view(); + emiss_sectors.push_back(f); + remapper->register_field_from_tgt(f); + } + + remapper->registration_ends(); + + return remapper; +} // create_horiz_remapper + +template +std::shared_ptr +srfEmissFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::vector §or_names, + const std::string &map_file) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit " + "the model. We only allow\n" + "srfEmiss data to be at the same or coarser resolution as " + "the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG( + map_file != "", + "ERROR: srfEmiss data is on a different grid than the model one,\n" + "but srfEmiss_remap_file is missing from srfEmiss parameter " + "list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + std::vector emiss_sectors; + + for(int icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); @@ -280,10 +343,35 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate template -template +template +void srfEmissFunctions::init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, + const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader) { + // Init horizontal remap + SrfEmissHorizInterp = + create_horiz_remapper(grid, data_file, sectors, srf_map_file); + + // Initialize the size of start/end/out data structures + SrfEmissData_start = srfEmissInput(ncol, numSectors); + SrfEmissData_end = srfEmissInput(ncol, numSectors); + SrfEmissData_out.init(ncol, 1, true); + + // Create reader (an AtmosphereInput object) + SrfEmissDataReader = + create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); +} // init_srf_emiss_objects + +template void srfEmissFunctions::init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, - const std::string &data_file, const std::array §ors, + const std::string &data_file, const std::vector §ors, const std::string &srf_map_file, // output std::shared_ptr &SrfEmissHorizInterp, @@ -295,8 +383,8 @@ void srfEmissFunctions::init_srf_emiss_objects( create_horiz_remapper(grid, data_file, sectors, srf_map_file); // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, FN); - SrfEmissData_end = srfEmissInput(ncol, FN); + SrfEmissData_start = srfEmissInput(ncol, sectors.size()); + SrfEmissData_end = srfEmissInput(ncol, sectors.size()); SrfEmissData_out.init(ncol, 1, true); // Create reader (an AtmosphereInput object) From fecc309be5b91f42351ffd39c13f087910793f5e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 21:06:17 -0700 Subject: [PATCH 29/65] The vector for surface emissions is working in a for loop --- ...and_online_emissions_process_interface.cpp | 721 +++++++----------- ...and_online_emissions_process_interface.hpp | 17 +- 2 files changed, 279 insertions(+), 459 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index d855a292926..eeb1b642cf2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -48,227 +48,103 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file std::string srf_map_file = m_params.get("srf_remap_file"); - // EXPERIMENTAL - /*using srf_emission_variant = - std::variant, srf_emiss<2>, srf_emiss<4>, srf_emiss<6>, - srf_emiss<8>, srf_emiss<16>>; - - std::vector srfballivec; - - static constexpr int dms_num_sectors1 = 1; - srf_emiss dms1; - dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms1.sectors = {"DMS"}; - - srfballivec.push_back(dms1); - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, srfballivec[0].data_file , srfballivec[0].sectors, - srf_map_file, - // output - srfballivec[0].HorizInterp_, srfballivec[0].EmissData_start_, - srfballivec[0].EmissData_end_, srfballivec[0].EmissData_out_, - srfballivec[0].EmissDataReader_);*/ - - std::vector srfballivec; - srf_emiss dms1; - dms1.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms1.sectors = {"DMS"}; - srfballivec.push_back(dms1); - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, srfballivec[0].data_file, srfballivec[0].sectors, - srf_map_file, - // output - srfballivec[0].HorizInterp_, srfballivec[0].Data_start_, - srfballivec[0].Data_end_, srfballivec[0].Data_out_, - srfballivec[0].DataReader_); - //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string dms_data_file = - m_params.get("srf_emis_specifier_for_DMS"); - - // Number of sectors - static constexpr int dms_num_sectors = 1; - - // Sector names in file - std::array dms_sectors = {"DMS"}; + srf_emiss dms; - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, dms_data_file, dms_sectors, srf_map_file, - // output - dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, - dmsSrfEmissData_out_, dmsSrfEmissDataReader_); + // File name and sectors + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.sectors = {"DMS"}; + srf_emiss_species_.push_back(dms); // add to the vector //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so2_data_file = - m_params.get("srf_emis_specifier_for_SO2"); - - // Number of sectors - static constexpr int so2_num_sectors = 6; - - // Sector names in file - std::array so2_sectors = {"AGR", "RCO", "SHP", - "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so2_data_file, so2_sectors, srf_map_file, - // output - so2SrfEmissHorizInterp_, so2SrfEmissData_start_, so2SrfEmissData_end_, - so2SrfEmissData_out_, so2SrfEmissDataReader_); - + srf_emiss so2; + // File name and sectors + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(so2); // add to the vector //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string bc_a4_data_file = - m_params.get("srf_emis_specifier_for_bc_a4"); - - // Number of sectors - static constexpr int bc_a4_num_sectors = 8; - - // Sector names in file - std::array bc_a4_sectors = { - "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, bc_a4_data_file, bc_a4_sectors, srf_map_file, - // output - bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_, bc_a4SrfEmissDataReader_); + srf_emiss bc_a4; + // File name and sectors + bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(bc_a4); // add to the vector //-------------------------------------------------------------------- // Init num_a1 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a1_data_file = - m_params.get("srf_emis_specifier_for_num_a1"); - - // Number of sectors - static constexpr int num_a1_num_sectors = 4; - - // Sector names in file - std::array num_a1_sectors = { - "num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", "num_a1_SO4_WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a1_data_file, num_a1_sectors, srf_map_file, - // output - num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_, - num_a1SrfEmissDataReader_); + srf_emiss num_a1; + // File name and sectors + num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; + srf_emiss_species_.push_back(num_a1); // add to the vector //-------------------------------------------------------------------- // Init num_a2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a2_data_file = - m_params.get("srf_emis_specifier_for_num_a2"); - - // Number of sectors - static constexpr int num_a2_num_sectors = 2; - - // Sector names in file - std::array num_a2_sectors = { - "num_a2_SO4_RCO", "num_a2_SO4_TRA"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a2_data_file, num_a2_sectors, srf_map_file, - // output - num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_, - num_a2SrfEmissDataReader_); + srf_emiss num_a2; + // File name and sectors + num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + srf_emiss_species_.push_back(num_a2); // add to the vector //-------------------------------------------------------------------- // Init num_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string num_a4_data_file = - m_params.get("srf_emis_specifier_for_num_a4"); - - // Number of sectors - static constexpr int num_a4_num_sectors = 16; - - // Sector names in file - std::array num_a4_sectors = { - "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", - "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", - "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", - "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, num_a4_data_file, num_a4_sectors, srf_map_file, - // output - num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_, - num_a4SrfEmissDataReader_); + srf_emiss num_a4; + // File name and sectors + num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + srf_emiss_species_.push_back(num_a4); // add to the vector //-------------------------------------------------------------------- // Init pom_a4 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string pom_a4_data_file = - m_params.get("srf_emis_specifier_for_pom_a4"); - - // Number of sectors - static constexpr int pom_a4_num_sectors = 8; - - // Sector names in file - std::array pom_a4_sectors = { - "AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, pom_a4_data_file, pom_a4_sectors, srf_map_file, - // output - pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_, - pom_a4SrfEmissDataReader_); + srf_emiss pom_a4; + // File name and sectors + pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); + pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(pom_a4); // add to the vector //-------------------------------------------------------------------- // Init so4_a1 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so4_a1_data_file = - m_params.get("srf_emis_specifier_for_so4_a1"); - - // Number of sectors - static constexpr int so4_a1_num_sectors = 4; - - // Sector names in file - std::array so4_a1_sectors = {"AGR", "SHP", - "SLV", "WST"}; - - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so4_a1_data_file, so4_a1_sectors, srf_map_file, - // output - so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_, - so4_a1SrfEmissDataReader_); + srf_emiss so4_a1; + // File name and sectors + so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + srf_emiss_species_.push_back(so4_a1); //-------------------------------------------------------------------- // Init so4_a2 srf emiss data structures //-------------------------------------------------------------------- - // File name - std::string so4_a2_data_file = - m_params.get("srf_emis_specifier_for_so4_a2"); + srf_emiss so4_a2; + // File name and sectors + so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); + so4_a2.sectors = {"RCO", "TRA"}; + srf_emiss_species_.push_back(so4_a2); - // Number of sectors - static constexpr int so4_a2_num_sectors = 2; - - // Sector names in file - std::array so4_a2_sectors = {"RCO", "TRA"}; + //-------------------------------------------------------------------- + // Init data structures to read and interpolate + //-------------------------------------------------------------------- - srfEmissFunc::init_srf_emiss_objects( - ncol_, grid_, so4_a2_data_file, so4_a2_sectors, srf_map_file, - // output - so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_, - so4_a2SrfEmissDataReader_); + for(const sr_emiss &ispec_srf : srf_emiss_species_) { + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, + // output + ispec_srf.HorizInterp_, ispec_srf.Data_start_, ispec_srf.Data_end_, + ispec_srf.Data_out_, ispec_srf.DataReader_); + } } // set_grid @@ -317,70 +193,12 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // and srfEmiss_end will be reloaded from file with the new month. //-------------------------------------------------------------------- - // Update dms srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - dmsSrfEmissDataReader_, timestamp(), curr_month, *dmsSrfEmissHorizInterp_, - dmsSrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so2SrfEmissDataReader_, timestamp(), curr_month, *so2SrfEmissHorizInterp_, - so2SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update bc_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - bc_a4SrfEmissDataReader_, timestamp(), curr_month, - *bc_a4SrfEmissHorizInterp_, bc_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a1 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a1SrfEmissDataReader_, timestamp(), curr_month, - *num_a1SrfEmissHorizInterp_, num_a1SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a2SrfEmissDataReader_, timestamp(), curr_month, - *num_a2SrfEmissHorizInterp_, num_a2SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update num_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - num_a4SrfEmissDataReader_, timestamp(), curr_month, - *num_a4SrfEmissHorizInterp_, num_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update pom_a4 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - pom_a4SrfEmissDataReader_, timestamp(), curr_month, - *pom_a4SrfEmissHorizInterp_, pom_a4SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so4_a1 srf emiss from file + // Update surface emissions from file //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so4_a1SrfEmissDataReader_, timestamp(), curr_month, - *so4_a1SrfEmissHorizInterp_, so4_a1SrfEmissData_end_); - - //-------------------------------------------------------------------- - // Update so4_a2 srf emiss from file - //-------------------------------------------------------------------- - srfEmissFunc::update_srfEmiss_data_from_file( - so4_a2SrfEmissDataReader_, timestamp(), curr_month, - *so4_a2SrfEmissHorizInterp_, so4_a2SrfEmissData_end_); - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << bc_a4SrfEmissData_end_.data.emiss_sectors[7](i) - << ":" << i << std::endl; + for(auto &ispec_srf : srf_emiss_species_) { + srfEmissFunc::update_srfEmiss_data_from_file( + ispec_srf.DataReader_, timestamp(), curr_month, *ispec_srf.HorizInterp_, + ispec_srf.Data_end_); } //----------------------------------------------------------------- @@ -407,211 +225,218 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- // Interpolate dms srf emiss data //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - dmsSrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - dmsSrfEmissDataReader_, ts, *dmsSrfEmissHorizInterp_, - dmsSrfEmissTimeState_, dmsSrfEmissData_start_, dmsSrfEmissData_end_); + for(auto &ispec_srf : srf_emiss_species_) { + // Update srfEmissTimeState, note the addition of dt + ispec_srf.TimeState_.t_now = ts.frac_of_year_in_days(); - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(dmsSrfEmissTimeState_, dmsSrfEmissData_start_, - dmsSrfEmissData_end_, dmsSrfEmissData_out_); + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + ispec_srf.DataReader_, ts, *ispec_srf.HorizInterp_, + ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_); - // update flux - auto constituent_fluxes_DMS = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(constituent_fluxes_DMS, - dmsSrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, - so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, - so2SrfEmissData_end_, so2SrfEmissData_out_); - // update flux - auto constituent_fluxes_SO2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::SO2)); - Kokkos::deep_copy(constituent_fluxes_SO2, - so2SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate bc_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, - bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_bc_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::bc_a4)); - Kokkos::deep_copy(constituent_fluxes_bc_a4, - bc_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a1 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, - num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, - num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a1)); - Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, - num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, - num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a2)); - Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate num_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, - num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, - num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_num_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a4)); - Kokkos::deep_copy(constituent_fluxes_num_a4, - num_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate pom_a4 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, - pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, - pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); - // update flux - auto constituent_fluxes_pom_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::pom_a4)); - Kokkos::deep_copy(constituent_fluxes_pom_a4, - pom_a4SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so4_a1 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, - so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, - so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); - // update flux - auto constituent_fluxes_so4_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a1)); - Kokkos::deep_copy(constituent_fluxes_so4_a1, - so4_a1SrfEmissData_out_.emiss_sectors[0]); - - //-------------------------------------------------------------------- - // Interpolate so4_a2 srf emiss data - //-------------------------------------------------------------------- - // Update srfEmissTimeState, note the addition of dt - so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - - // Update time state and if the month has changed, update the data. - srfEmissFunc::update_srfEmiss_timestate( - so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, - so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_); - - // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, - so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); - // update flux - auto constituent_fluxes_so4_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a2)); - Kokkos::deep_copy(constituent_fluxes_so4_a2, - so4_a2SrfEmissData_out_.emiss_sectors[0]); - - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] - << std::endl; + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, + ispec_srf.Data_end_, ispec_srf.Data_out_); + // update flux + auto clfx = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(cflx, dmsSrfEmissData_out_.emiss_sectors[0]); } + /* +// update flux +auto constituent_fluxes_DMS = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); +Kokkos::deep_copy(constituent_fluxes_DMS, + dmsSrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, + so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, + so2SrfEmissData_end_, so2SrfEmissData_out_); +// update flux +auto constituent_fluxes_SO2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::SO2)); +Kokkos::deep_copy(constituent_fluxes_SO2, + so2SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate bc_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, + bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, + bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_bc_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::bc_a4)); +Kokkos::deep_copy(constituent_fluxes_bc_a4, + bc_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a1 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, + num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, + num_a1SrfEmissData_start_, + num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a1)); +Kokkos::deep_copy(constituent_fluxes_num_a1, + num_a1SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, + num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, + num_a2SrfEmissData_start_, + num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a2)); +Kokkos::deep_copy(constituent_fluxes_num_a2, + num_a2SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate num_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, + num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, + num_a4SrfEmissData_start_, + num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_num_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::num_a4)); +Kokkos::deep_copy(constituent_fluxes_num_a4, + num_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate pom_a4 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, + pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, + pom_a4SrfEmissData_start_, + pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); +// update flux +auto constituent_fluxes_pom_a4 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::pom_a4)); +Kokkos::deep_copy(constituent_fluxes_pom_a4, + pom_a4SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so4_a1 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, + so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, + so4_a1SrfEmissData_start_, + so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); +// update flux +auto constituent_fluxes_so4_a1 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a1)); +Kokkos::deep_copy(constituent_fluxes_so4_a1, + so4_a1SrfEmissData_out_.emiss_sectors[0]); + +//-------------------------------------------------------------------- +// Interpolate so4_a2 srf emiss data +//-------------------------------------------------------------------- +// Update srfEmissTimeState, note the addition of dt +so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); + +// Update time state and if the month has changed, update the data. +srfEmissFunc::update_srfEmiss_timestate( + so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, + so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_); + +// Call the main srfEmiss routine to get interpolated aerosol forcings. +srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, + so4_a2SrfEmissData_start_, + so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); +// update flux +auto constituent_fluxes_so4_a2 = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::so4_a2)); +Kokkos::deep_copy(constituent_fluxes_so4_a2, + so4_a2SrfEmissData_out_.emiss_sectors[0]); + +for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" + << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] + << std::endl; +}*/ /* Rough notes: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index dca30a176dd..2b321571da2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -127,27 +127,22 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // A struct carrying all the fields needed to read // surface emissions of a species - // template struct srf_emiss { - // srf_emiss(int n){numSectors=n;} - // data file name + // Data file name std::string data_file; - // static int numSectors; - // Sector names in file + // Sector names in file std::vector sectors; - + // Data structure for reading interpolation std::shared_ptr HorizInterp_; std::shared_ptr DataReader_; srfEmissFunc::srfEmissTimeState TimeState_; srfEmissFunc::srfEmissInput Data_start_, Data_end_; srfEmissFunc::srfEmissOutput Data_out_; - - /*srfEmissFunc::init_srf_emiss_objects( - // output - dmsSrfEmissHorizInterp_, dmsSrfEmissData_start_, dmsSrfEmissData_end_, - dmsSrfEmissData_out_, dmsSrfEmissDataReader_);*/ }; + // A vector for carrying emissions for all the species + std::vector srf_emiss_species_; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; From 4f816563a6f71fa1edfe59b99be29825bf010ddb Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 31 Jul 2024 21:15:19 -0700 Subject: [PATCH 30/65] Fixes compilation issues in previous commit --- ...mam_srf_and_online_emissions_process_interface.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index eeb1b642cf2..17481dfc4d0 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -137,8 +137,7 @@ void MAMSrfOnlineEmiss::set_grids( //-------------------------------------------------------------------- // Init data structures to read and interpolate //-------------------------------------------------------------------- - - for(const sr_emiss &ispec_srf : srf_emiss_species_) { + for(srf_emiss &ispec_srf : srf_emiss_species_) { srfEmissFunc::init_srf_emiss_objects( ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, // output @@ -239,9 +238,11 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_, ispec_srf.Data_out_); // update flux - auto clfx = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(cflx, dmsSrfEmissData_out_.emiss_sectors[0]); + /*auto clfx = + Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), + static_cast(spcIndex_in_pcnst::DMS)); + Kokkos::deep_copy(cflx, + dmsSrfEmissData_out_.emiss_sectors[0]);*/ } /* // update flux From f3aad49cac2c7a334b105b0fd73493b29b4c9f8d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 07:21:47 -0700 Subject: [PATCH 31/65] Updates constituent flux array with the righ units --- ...and_online_emissions_process_interface.cpp | 349 +++++------------- ...and_online_emissions_process_interface.hpp | 111 ++---- 2 files changed, 123 insertions(+), 337 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 17481dfc4d0..01167845fd5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -51,98 +51,106 @@ void MAMSrfOnlineEmiss::set_grids( //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- - srf_emiss dms; - - // File name and sectors - dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); - dms.sectors = {"DMS"}; + srf_emiss_ dms; + // File name, name and sectors + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.species_name = "dms"; + dms.sectors = {"DMS"}; srf_emiss_species_.push_back(dms); // add to the vector //-------------------------------------------------------------------- // Init so2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so2; - // File name and sectors - so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); - so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_ so2; + // File name, name and sectors + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.species_name = "so2"; + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(so2); // add to the vector //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss bc_a4; - // File name and sectors + srf_emiss_ bc_a4; + // File name, name and sectors bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); - bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + bc_a4.species_name = "bc_a4"; + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(bc_a4); // add to the vector //-------------------------------------------------------------------- // Init num_a1 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a1; - // File name and sectors + srf_emiss_ num_a1; + // File name, name and sectors num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); - num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", - "num_a1_SO4_WST"}; + num_a1.species_name = "num_a1"; + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; srf_emiss_species_.push_back(num_a1); // add to the vector //-------------------------------------------------------------------- // Init num_a2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a2; - // File name and sectors + srf_emiss_ num_a2; + // File name, name and sectors num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); - num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + num_a2.species_name = "num_a2"; + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; srf_emiss_species_.push_back(num_a2); // add to the vector //-------------------------------------------------------------------- // Init num_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss num_a4; - // File name and sectors + srf_emiss_ num_a4; + // File name, name and sectors num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); - num_a4.sectors = { - "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", - "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", - "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", - "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + num_a4.species_name = "num_a4"; + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; srf_emiss_species_.push_back(num_a4); // add to the vector //-------------------------------------------------------------------- // Init pom_a4 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss pom_a4; - // File name and sectors + srf_emiss_ pom_a4; + // File name, name and sectors pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); - pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + pom_a4.species_name = "pom_a4"; + pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(pom_a4); // add to the vector //-------------------------------------------------------------------- // Init so4_a1 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so4_a1; - // File name and sectors + srf_emiss_ so4_a1; + // File name, name and sectors so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); - so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + so4_a1.species_name = "so4_a1"; + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; srf_emiss_species_.push_back(so4_a1); //-------------------------------------------------------------------- // Init so4_a2 srf emiss data structures //-------------------------------------------------------------------- - srf_emiss so4_a2; - // File name and sectors + srf_emiss_ so4_a2; + // File name, name and sectors so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); - so4_a2.sectors = {"RCO", "TRA"}; + so4_a2.species_name = "so4_a2"; + so4_a2.sectors = {"RCO", "TRA"}; srf_emiss_species_.push_back(so4_a2); //-------------------------------------------------------------------- // Init data structures to read and interpolate //-------------------------------------------------------------------- - for(srf_emiss &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::init_srf_emiss_objects( ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, // output - ispec_srf.HorizInterp_, ispec_srf.Data_start_, ispec_srf.Data_end_, - ispec_srf.Data_out_, ispec_srf.DataReader_); + ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, + ispec_srf.data_out_, ispec_srf.dataReader_); } } // set_grid @@ -183,6 +191,13 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); + // --------------------------------------------------------------- + // Allocate memory for local and work arrays + // --------------------------------------------------------------- + + // work array to store fluxes after unit conversions to kg/m2/s + fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); + // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; @@ -194,10 +209,10 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { //-------------------------------------------------------------------- // Update surface emissions from file //-------------------------------------------------------------------- - for(auto &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::update_srfEmiss_data_from_file( - ispec_srf.DataReader_, timestamp(), curr_month, *ispec_srf.HorizInterp_, - ispec_srf.Data_end_); + ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, + ispec_srf.data_end_); } //----------------------------------------------------------------- @@ -218,232 +233,68 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + // policy to loop over columns only + const auto ncol_only_policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, 1); + // Gather time and state information for interpolation auto ts = timestamp() + dt; //-------------------------------------------------------------------- - // Interpolate dms srf emiss data + // Interpolate srf emiss data //-------------------------------------------------------------------- - for(auto &ispec_srf : srf_emiss_species_) { + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { // Update srfEmissTimeState, note the addition of dt - ispec_srf.TimeState_.t_now = ts.frac_of_year_in_days(); + ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( - ispec_srf.DataReader_, ts, *ispec_srf.HorizInterp_, - ispec_srf.TimeState_, ispec_srf.Data_start_, ispec_srf.Data_end_); + ispec_srf.dataReader_, ts, *ispec_srf.horizInterp_, + ispec_srf.timeState_, ispec_srf.data_start_, ispec_srf.data_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. - srfEmissFunc::srfEmiss_main(ispec_srf.TimeState_, ispec_srf.Data_start_, - ispec_srf.Data_end_, ispec_srf.Data_out_); - // update flux - /*auto clfx = + srfEmissFunc::srfEmiss_main(ispec_srf.timeState_, ispec_srf.data_start_, + ispec_srf.data_end_, ispec_srf.data_out_); + + //-------------------------------------------------------------------- + // Modify units to MKS units (from molecules/cm2/s to kg/m2/s) + //-------------------------------------------------------------------- + // Get species index in array with pcnst dimension (e.g., state_q or + // constituent_fluxes_) + const int species_index = spcIndex_in_pcnst_.at(ispec_srf.species_name); + + // modify units from molecules/cm2/s to kg/m2/s + const Real mfactor = + amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; + // Parallel loop over all the columns to update units + Kokkos::parallel_for( + ncol_only_policy, + KOKKOS_LAMBDA(const MAMSrfOnlineEmiss::KT::MemberType &team) { + const int icol = team.league_rank(); + fluxes_in_mks_units_(icol) = + ispec_srf.data_out_.emiss_sectors[0](icol) * mfactor; + }); + + // Get subview + auto flux_1d_view = Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); - Kokkos::deep_copy(cflx, - dmsSrfEmissData_out_.emiss_sectors[0]);*/ + spcIndex_in_pcnst_.at(ispec_srf.species_name)); + + // update flux in constituent_fluxes_ view + Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); + } + + for(int i = 19; i < 30; ++i) { + std::cout << "BALLI:" << srf_emiss_species_[8].data_out_.emiss_sectors[0](i) + << ":" << i << ":" << constituent_fluxes_(i, 37) << std::endl; } - /* -// update flux -auto constituent_fluxes_DMS = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::DMS)); -Kokkos::deep_copy(constituent_fluxes_DMS, - dmsSrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so2SrfEmissDataReader_, ts, *so2SrfEmissHorizInterp_, - so2SrfEmissTimeState_, so2SrfEmissData_start_, so2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so2SrfEmissTimeState_, so2SrfEmissData_start_, - so2SrfEmissData_end_, so2SrfEmissData_out_); -// update flux -auto constituent_fluxes_SO2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::SO2)); -Kokkos::deep_copy(constituent_fluxes_SO2, - so2SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate bc_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -bc_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - bc_a4SrfEmissDataReader_, ts, *bc_a4SrfEmissHorizInterp_, - bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(bc_a4SrfEmissTimeState_, bc_a4SrfEmissData_start_, - bc_a4SrfEmissData_end_, bc_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_bc_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::bc_a4)); -Kokkos::deep_copy(constituent_fluxes_bc_a4, - bc_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a1 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a1SrfEmissDataReader_, ts, *num_a1SrfEmissHorizInterp_, - num_a1SrfEmissTimeState_, num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a1SrfEmissTimeState_, - num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_, num_a1SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a1)); -Kokkos::deep_copy(constituent_fluxes_num_a1, - num_a1SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a2SrfEmissDataReader_, ts, *num_a2SrfEmissHorizInterp_, - num_a2SrfEmissTimeState_, num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a2SrfEmissTimeState_, - num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_, num_a2SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a2)); -Kokkos::deep_copy(constituent_fluxes_num_a2, - num_a2SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate num_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -num_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - num_a4SrfEmissDataReader_, ts, *num_a4SrfEmissHorizInterp_, - num_a4SrfEmissTimeState_, num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(num_a4SrfEmissTimeState_, - num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_, num_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_num_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::num_a4)); -Kokkos::deep_copy(constituent_fluxes_num_a4, - num_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate pom_a4 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -pom_a4SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - pom_a4SrfEmissDataReader_, ts, *pom_a4SrfEmissHorizInterp_, - pom_a4SrfEmissTimeState_, pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(pom_a4SrfEmissTimeState_, - pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_, pom_a4SrfEmissData_out_); -// update flux -auto constituent_fluxes_pom_a4 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::pom_a4)); -Kokkos::deep_copy(constituent_fluxes_pom_a4, - pom_a4SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so4_a1 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so4_a1SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so4_a1SrfEmissDataReader_, ts, *so4_a1SrfEmissHorizInterp_, - so4_a1SrfEmissTimeState_, so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so4_a1SrfEmissTimeState_, - so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_, so4_a1SrfEmissData_out_); -// update flux -auto constituent_fluxes_so4_a1 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a1)); -Kokkos::deep_copy(constituent_fluxes_so4_a1, - so4_a1SrfEmissData_out_.emiss_sectors[0]); - -//-------------------------------------------------------------------- -// Interpolate so4_a2 srf emiss data -//-------------------------------------------------------------------- -// Update srfEmissTimeState, note the addition of dt -so4_a2SrfEmissTimeState_.t_now = ts.frac_of_year_in_days(); - -// Update time state and if the month has changed, update the data. -srfEmissFunc::update_srfEmiss_timestate( - so4_a2SrfEmissDataReader_, ts, *so4_a2SrfEmissHorizInterp_, - so4_a2SrfEmissTimeState_, so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_); - -// Call the main srfEmiss routine to get interpolated aerosol forcings. -srfEmissFunc::srfEmiss_main(so4_a2SrfEmissTimeState_, - so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_, so4_a2SrfEmissData_out_); -// update flux -auto constituent_fluxes_so4_a2 = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - static_cast(spcIndex_in_pcnst::so4_a2)); -Kokkos::deep_copy(constituent_fluxes_so4_a2, - so4_a2SrfEmissData_out_.emiss_sectors[0]); - -for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << so4_a2SrfEmissData_out_.emiss_sectors[0](i) << ":" - << i << ":" << mam4::gas_chemistry::adv_mass[21 - offset_] - << std::endl; -}*/ /* Rough notes: Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions and - set_srf_emissions. + chemistry.F90. Basically call two subroutines, aero_model_emissions + and set_srf_emissions. Here is the code: diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 2b321571da2..1669125b9fb 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -24,6 +24,7 @@ namespace scream { // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; using view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels @@ -44,6 +45,12 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { view_2d constituent_fluxes_; + // work array to store fluxes after unit conversions to kg/m2/s + view_1d fluxes_in_mks_units_; + + // Unified atomic mass unit used for unit conversion (BAD constant) + static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + public: using srfEmissFunc = mam_coupling::srfEmissFunctions; @@ -112,110 +119,38 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // preprocessing scratch pad Preprocess preprocess_; - // Species index in tracer array with "pcnst" indices - enum class spcIndex_in_pcnst : int { - SO2 = 12, - DMS = 13, - so4_a1 = 15, - num_a1 = 22, - so4_a2 = 23, - num_a2 = 27, - pom_a4 = 36, - bc_a4 = 37, - num_a4 = 39, - }; + // Species index (zero-based) in tracer array with "pcnst" dimension + // FIXME: Remove the hardwired indices and use a function + // to find them from an array. + const std::map spcIndex_in_pcnst_ = { + {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, + {"num_a1", 22}, {"so4_a2", 23}, {"num_a2", 27}, + {"pom_a4", 36}, {"bc_a4", 37}, {"num_a4", 39}}; // A struct carrying all the fields needed to read // surface emissions of a species - struct srf_emiss { + struct srf_emiss_ { + // species name + std::string species_name; // Data file name std::string data_file; // Sector names in file std::vector sectors; // Data structure for reading interpolation - std::shared_ptr HorizInterp_; - std::shared_ptr DataReader_; - srfEmissFunc::srfEmissTimeState TimeState_; - srfEmissFunc::srfEmissInput Data_start_, Data_end_; - srfEmissFunc::srfEmissOutput Data_out_; + std::shared_ptr horizInterp_; + std::shared_ptr dataReader_; + srfEmissFunc::srfEmissTimeState timeState_; + srfEmissFunc::srfEmissInput data_start_, data_end_; + srfEmissFunc::srfEmissOutput data_out_; }; // A vector for carrying emissions for all the species - std::vector srf_emiss_species_; + std::vector srf_emiss_species_; // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; - // Data structures to read DMS data file - std::shared_ptr dmsSrfEmissHorizInterp_; - std::shared_ptr dmsSrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState dmsSrfEmissTimeState_; - srfEmissFunc::srfEmissInput dmsSrfEmissData_start_, dmsSrfEmissData_end_; - srfEmissFunc::srfEmissOutput dmsSrfEmissData_out_; - - // Data structures to read so2 data file - std::shared_ptr so2SrfEmissHorizInterp_; - std::shared_ptr so2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so2SrfEmissData_start_, so2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so2SrfEmissData_out_; - - // Data structures to read bc_a4 data file - std::shared_ptr bc_a4SrfEmissHorizInterp_; - std::shared_ptr bc_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState bc_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput bc_a4SrfEmissData_start_, bc_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput bc_a4SrfEmissData_out_; - - // Data structures to read num_a1 data file - std::shared_ptr num_a1SrfEmissHorizInterp_; - std::shared_ptr num_a1SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a1SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a1SrfEmissData_start_, - num_a1SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a1SrfEmissData_out_; - - // Data structures to read num_a2 data file - std::shared_ptr num_a2SrfEmissHorizInterp_; - std::shared_ptr num_a2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a2SrfEmissData_start_, - num_a2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a2SrfEmissData_out_; - - // Data structures to read num_a4 data file - std::shared_ptr num_a4SrfEmissHorizInterp_; - std::shared_ptr num_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState num_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput num_a4SrfEmissData_start_, - num_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput num_a4SrfEmissData_out_; - - // Data structures to read pom_a4 data file - std::shared_ptr pom_a4SrfEmissHorizInterp_; - std::shared_ptr pom_a4SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState pom_a4SrfEmissTimeState_; - srfEmissFunc::srfEmissInput pom_a4SrfEmissData_start_, - pom_a4SrfEmissData_end_; - srfEmissFunc::srfEmissOutput pom_a4SrfEmissData_out_; - - // Data structures to read so4_a1 data file - std::shared_ptr so4_a1SrfEmissHorizInterp_; - std::shared_ptr so4_a1SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so4_a1SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so4_a1SrfEmissData_start_, - so4_a1SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so4_a1SrfEmissData_out_; - - // Data structures to read so4_a2 data file - std::shared_ptr so4_a2SrfEmissHorizInterp_; - std::shared_ptr so4_a2SrfEmissDataReader_; - srfEmissFunc::srfEmissTimeState so4_a2SrfEmissTimeState_; - srfEmissFunc::srfEmissInput so4_a2SrfEmissData_start_, - so4_a2SrfEmissData_end_; - srfEmissFunc::srfEmissOutput so4_a2SrfEmissData_out_; - }; // MAMSrfOnlineEmiss } // namespace scream From 5b35e31791083141bd4e2ee18b4b54be2548d168 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 13:02:03 -0700 Subject: [PATCH 32/65] Adds constituent flixes interface and a test --- .../eamxx/src/physics/mam/CMakeLists.txt | 3 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 231 ++++++++++++++++++ ...eamxx_mam_constituent_fluxes_interface.hpp | 76 ++++++ .../eamxx/src/physics/mam/mam_coupling.hpp | 18 ++ .../eamxx/src/physics/register_physics.hpp | 2 + .../eamxx/tests/single-process/CMakeLists.txt | 1 + .../mam/constituent_fluxes/CMakeLists.txt | 44 ++++ .../mam/constituent_fluxes/input.yaml | 37 +++ .../mam/constituent_fluxes/output.yaml | 12 + 9 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml create mode 100644 components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 9138bba6d5a..9874f79f9eb 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -46,7 +46,8 @@ add_library(mam eamxx_mam_dry_deposition_process_interface.cpp eamxx_mam_aci_process_interface.cpp eamxx_mam_wetscav_process_interface.cpp - eamxx_mam_srf_and_online_emissions_process_interface.cpp) + eamxx_mam_srf_and_online_emissions_process_interface.cpp + eamxx_mam_constituent_fluxes_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp new file mode 100644 index 00000000000..91d587c31d3 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -0,0 +1,231 @@ +#include +namespace scream { + +// ================================================================ +// Constructor +// ================================================================ +MAMConstituentFluxes::MAMConstituentFluxes(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ================================================================ +// SET_GRIDS +// ================================================================ +void MAMConstituentFluxes::set_grids( + const std::shared_ptr grids_manager) { + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + + static constexpr int pcnst = mam4::aero_model::pcnst; + + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + static constexpr Units m2(m * m, "m2"); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + grid_name); + + // --------------------------------------------------------------------- + // These variables are "Updated" or inputs/outputs for the process + // --------------------------------------------------------------------- + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + +} // set_grid + +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMConstituentFluxes::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializes the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMConstituentFluxes::init_buffers( + const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMConstituentFluxes."); +} + +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ +void MAMConstituentFluxes::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Inputs + // --------------------------------------------------------------- + constituent_fluxes_ = + get_field_in("constituent_fluxes").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + //----------------------------------------------------------------- + // Allocate memory + //----------------------------------------------------------------- + + rpdel_ = view_2d("rpdel_", ncol_, nlev_); + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + // preprocess_.initialize(constituent_fluxes_); + +} // end initialize_impl() + +// ================================================================ +// RUN_IMPL +// ================================================================ +void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + /*FORTRAN CODE: + What we need: + 1. ncol + 2. all tracers + 3. timestep (verify what is rztodt??) + 4. gravit, rpdel + 5. cflx + 6. Ensure dry/et-wet/dry isconsistent....comment on that!! + + + ncol = state%ncol + + !------------------------------------------------------- + ! Assume 'wet' mixing ratios in surface diffusion code. + ! don't convert co2 tracers to wet mixing ratios + + cnst_type_loc(:) = cnst_type(:) + call set_dry_to_wet(state, cnst_type_loc) + + !------------------------------------------------------- + ! Initialize ptend + + lq(:) = .TRUE. + call physics_ptend_init(ptend, state%psetcols, 'clubb_srf', lq=lq) + + !------------------------------------------------------- + ! Calculate tracer mixing ratio tendencies from cflx + + rztodt = 1._r8/ztodt + ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) + tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver) + + do m = 2, pcnst + ptend%q(:ncol,pver,m) = ptend%q(:ncol,pver,m) + tmp1(:ncol) * + cam_in%cflx(:ncol,m) enddo + + ptend%q(:ncol,:pver,:) = (ptend%q(:ncol,:pver,:) - state%q(:ncol,:pver,:)) * + rztodt + + ! Convert tendencies of dry constituents to dry basis. + do m = 1,pcnst + if (cnst_type(m).eq.'dry') then + ptend%q(:ncol,:pver,m) = + ptend%q(:ncol,:pver,m)*state%pdel(:ncol,:pver)/state%pdeldry(:ncol,:pver) + endif + end do + + !------------------------------------------------------- + ! convert wet mmr back to dry before conservation check + ! avoid converting co2 tracers again + + cnst_type_loc(:) = cnst_type(:) + call co2_cycle_set_cnst_type(cnst_type_loc, 'wet') + call set_wet_to_dry(state, cnst_type_loc) + + */ +} + +// ============================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp new file mode 100644 index 00000000000..0f17e5b0632 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -0,0 +1,76 @@ +#ifndef EAMXX_MAM_CONSTITUENT_FLUXES_HPP +#define EAMXX_MAM_CONSTITUENT_FLUXES_HPP + +// For declaring contituent fluxes class derived from atm process +// class +#include +// For MAM4 aerosol configuration +#include +#include + +namespace scream { + +// The process responsible for applying MAM4 constituent fluxes. The +// AD stores exactly ONE instance of this class in its list of subcomponents. +class MAMConstituentFluxes final : public scream::AtmosphereProcess { + using KT = ekat::KokkosTypes; + using view_2d = Field::view_dev_t; + using const_view_2d = Field::view_dev_t; + + // number of horizontal columns + int ncol_, nlev_; + + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + const_view_2d constituent_fluxes_; + + view_2d rpdel_; // Inverse of pdel_ or pseudo_density + + public: + // Constructor + MAMConstituentFluxes(const ekat::Comm &comm, + const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_constituent_fluxes"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMConstituentFluxes + +} // namespace scream + +#endif // EAMXX_MAM_CONSTITUENT_FLUXES_HPP diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index d490ab76155..09d64d6028d 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -776,6 +776,24 @@ void compute_wet_mixing_ratios(const Team& team, }); } +// Computes the reciprocal of pseudo density for a column +KOKKOS_INLINE_FUNCTION +void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, + const_view_2d pdel, + const int nlev, + // output + view_2d rpdel) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); + }); +} // Scream (or EAMxx) can sometimes extend views beyond model levels (nlev) as it uses // "packs". Following function copies a 2d view till model levels inline diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index f8d3a9edb19..d837e96311f 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -30,6 +30,7 @@ #include "physics/mam/eamxx_mam_aci_process_interface.hpp" #include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" #include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" +#include "physics/mam/eamxx_mam_constituent_fluxes_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -70,6 +71,7 @@ inline void register_physics () { proc_factory.register_product("mam4_aci",&create_atmosphere_process); proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); proc_factory.register_product("mam4_srf_online_emiss",&create_atmosphere_process); + proc_factory.register_product("mam4_constituent_fluxes",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index 62435db229f..40565d94bf8 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -24,6 +24,7 @@ if (SCREAM_ENABLE_MAM) add_subdirectory(mam/drydep) add_subdirectory(mam/wet_scav) add_subdirectory(mam/emissions) + add_subdirectory(mam/constituent_fluxes) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt new file mode 100644 index 00000000000..2615f152b26 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt @@ -0,0 +1,44 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_constituent_fluxes_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_constituent_fluxes physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_constituent_fluxes physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml new file mode 100644 index 00000000000..d012274d146 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml @@ -0,0 +1,37 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_constituent_fluxes] +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + pbl_height : 1.0 + constituent_fluxes: 1.e-5 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml new file mode 100644 index 00000000000..e51b96eb5ce --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml @@ -0,0 +1,12 @@ +%YAML 1.1 +--- +filename_prefix: mam4_constituent_fluxes_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - T_mid +output_control: + Frequency: 1 + frequency_units: nsteps +... From b8ff3601be8c9ceb91a08a63536405645b7226b9 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 19:39:46 -0700 Subject: [PATCH 33/65] Adds code to create progs and atm datastructures --- ...eamxx_mam_constituent_fluxes_interface.cpp | 190 ++++++++++++++++-- 1 file changed, 176 insertions(+), 14 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 91d587c31d3..6075e066b54 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -26,8 +26,11 @@ void MAMConstituentFluxes::set_grids( using namespace ekat::units; auto q_unit = kg / kg; // units of mass mixing ratios of tracers auto n_unit = 1 / kg; // units of number mixing ratios of tracers + auto nondim = ekat::units::Units::nondimensional(); + FieldLayout scalar2d = grid_->get_2d_scalar_layout(); FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); static constexpr int pcnst = mam4::aero_model::pcnst; @@ -37,44 +40,98 @@ void MAMConstituentFluxes::set_grids( // -------------------------------------------------------------------------- // These variables are "Required" or pure inputs for the process // -------------------------------------------------------------------------- + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg](Require only for building DS) + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg](Require only for building DS) + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg](Require only for building DS) + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + // Temperature[K] at midpoints add_field("T_mid", scalar3d_mid, K, grid_name); + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) + add_field("omega", scalar3d_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + // Layer thickness(pdel) [Pa] at midpoints add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + // Planetary boundary layer height [m] (Require only for building DS) + add_field("pbl_height", scalar2d, m, grid_name); + + // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + static constexpr Units m2(m * m, "m2"); + static constexpr Units s2(s * s, "s2"); + + // Surface geopotential [m2/s2] (Require only for building DS) + add_field("phis", scalar2d, m2 / s2, grid_name); + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); // --------------------------------------------------------------------- // These variables are "Updated" or inputs/outputs for the process // --------------------------------------------------------------------- - - // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing - // ratios - for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); - + // NOTE: Cloud borne aerosols are not updated in this process but are included + // to create datastructures. + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = - mam_coupling::int_aero_mmr_field_name(m, a); - + mam_coupling::int_aero_mmr_field_name(mode, a); if(strlen(int_mmr_field_name) > 0) { add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } - } - } + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(mode, a); + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); + } + } // end for loop num species + } // end for loop for num modes - // aerosol-related gases: mass mixing ratios for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); - } + } // end for loop num gases } // set_grid @@ -111,8 +168,42 @@ void MAMConstituentFluxes::init_buffers( // ================================================================ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- - // Inputs + // Input fields read in from IC file, namelist or other processes // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + // FIMXE: specifically look which among these are actually used by the process + wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + constituent_fluxes_ = get_field_in("constituent_fluxes").get_view(); dry_atm_.p_del = get_field_in("pseudo_density").get_view(); @@ -126,6 +217,12 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(int_nmr_field_name).get_view(); dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios const char *int_mmr_field_name = @@ -136,6 +233,15 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(int_mmr_field_name).get_view(); dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } } } for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { @@ -167,8 +273,65 @@ void MAMConstituentFluxes::run_impl(const double dt) { // preprocess input -- needs a scan for the calculation of atm height // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); // Kokkos::fence(); + + haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] + + mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, + nlev_, + // output + rpdel_); + // ------------------------------------------------------------------- + // NOTES: The following code is an adaptation of cflx.F90 code in E3SM + // In EAMxx, all constituents are considered "wet" (or have wet mixing + // ratios), so we are *not* doing any wet to dry conversions here. We + // are simply updating the MAM4xx tracers using the "constituent + // fluxes" + // ------------------------------------------------------------------- + + Real rztodt = 1.0 / dt; + + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + /////////dddd + // To construct state_q, we need fields from Prognostics (aerosols) + // and Atmosphere (water species such as qv, qc etc.) + + // get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(dry_aero_, icol); + + // get atmospheric quantities + haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); + + // Construct state_q (interstitial) and qqcw (cloud borne) arrays + /*constexpr auto pver = mam4::ndrop::pver; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pver), [&](int klev) + { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics(progs_at_col, haero_atm, + state_q_at_lev_col, + klev); + + // get the start index for aerosols species in the state_q array + int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + + // create colum views of state_q + for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { + state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; + } + });*/ + }); + ////////dddd + + /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) + tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ + /*FORTRAN CODE: What we need: 1. ncol @@ -183,7 +346,6 @@ void MAMConstituentFluxes::run_impl(const double dt) { !------------------------------------------------------- ! Assume 'wet' mixing ratios in surface diffusion code. - ! don't convert co2 tracers to wet mixing ratios cnst_type_loc(:) = cnst_type(:) call set_dry_to_wet(state, cnst_type_loc) From 552a785b600e3222d7f0e0ba263190f2e9761066 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 21:31:38 -0700 Subject: [PATCH 34/65] A working end-to-end code --- .../physics/mam/eamxx_mam_aci_functions.hpp | 2 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 58 +++++++++---------- ...eamxx_mam_constituent_fluxes_interface.hpp | 2 +- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 03b04ac739a..ab142fedc78 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -397,7 +397,7 @@ void call_function_dropmixnuc( progs_at_col, haero_atm, state_q_at_lev_col, klev); // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + int istart = mam4::utils::aero_start_ind(); // create colum views of state_q for(int icnst = istart; icnst < mam4::aero_model::pcnst; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 6075e066b54..54d863f8b6f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -215,13 +215,11 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); - dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; // cloudborne aerosol tracers of interest: number (n) mixing ratios const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); - dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios @@ -231,7 +229,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { if(strlen(int_mmr_field_name) > 0) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); - dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios @@ -240,7 +237,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { if(strlen(cld_mmr_field_name) > 0) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); - dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; } } } @@ -248,7 +244,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); - dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; } //----------------------------------------------------------------- @@ -289,9 +284,6 @@ void MAMConstituentFluxes::run_impl(const double dt) { // are simply updating the MAM4xx tracers using the "constituent // fluxes" // ------------------------------------------------------------------- - - Real rztodt = 1.0 / dt; - Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); @@ -301,33 +293,41 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get prognostics mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(dry_aero_, icol); + mam_coupling::aerosols_for_column(wet_aero_, icol); // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); // Construct state_q (interstitial) and qqcw (cloud borne) arrays - /*constexpr auto pver = mam4::ndrop::pver; - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pver), [&](int klev) - { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics(progs_at_col, haero_atm, - state_q_at_lev_col, - klev); - - // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; - - // create colum views of state_q - for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_work_loc(icol, klev, icnst) = state_q_at_lev_col[icnst]; - } - });*/ + constexpr auto pver = mam4::ndrop::pver; + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, pver), [&](int klev) { + Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; + + // get state_q at a grid cell (col,lev) + // NOTE: The order of species in state_q_at_lev_col + // is the same as in E3SM state%q array + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_lev_col, klev); + + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); + std::cout << "istart:" << mam4::utils::gasses_start_ind() + << std::endl; + // kg/m2/s to kg/kg/ + Real unit_factor = dt * gravit * rpdel_(icol, klev); + // Update state with the constituent fluxes (modified + // units) + for(int icnst = istart; icnst < mam4::aero_model::pcnst; + ++icnst) { + state_q_at_lev_col[icnst] = + state_q_at_lev_col[icnst] + + constituent_fluxes_(icol, icnst) * unit_factor; + } + mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, + progs_at_col, klev); + }); }); - ////////dddd /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 0f17e5b0632..02c4b879093 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -25,7 +25,7 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { mam_coupling::DryAtmosphere dry_atm_; // aerosol state variables - mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::AerosolState wet_aero_; //, dry_aero_; // buffer for sotring temporary variables mam_coupling::Buffer buffer_; From a3c0ab9dbf0c08c31407db8a31f316e92107160f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 1 Aug 2024 22:20:52 -0700 Subject: [PATCH 35/65] Verified change in DMS conc before and after update, moved aci rpdel function to mam_coupling --- .../physics/mam/eamxx_mam_aci_functions.hpp | 17 --- .../mam/eamxx_mam_aci_process_interface.cpp | 2 +- ...eamxx_mam_constituent_fluxes_interface.cpp | 117 ++++++------------ ...eamxx_mam_constituent_fluxes_interface.hpp | 41 ++++++ 4 files changed, 78 insertions(+), 99 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index ab142fedc78..8bb52c918d2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -198,23 +198,6 @@ void store_liquid_cloud_fraction( }); } -void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, - MAMAci::const_view_2d pdel, - const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { - EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), - "Error: pdel should be > 0.\n"); - rpdel(icol, kk) = 1 / pdel(icol, kk); - }); - }); -} - void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, const Real dt, mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e919aa066b4..7800a43c252 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -597,7 +597,7 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); - compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, + mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 54d863f8b6f..5c5129d1b7c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -84,6 +84,7 @@ void MAMConstituentFluxes::set_grids( // Surface geopotential [m2/s2] (Require only for building DS) add_field("phis", scalar2d, m2 / s2, grid_name); + // Constituent fluxes at the surface (gasses and aerosols) add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); @@ -91,7 +92,8 @@ void MAMConstituentFluxes::set_grids( // These variables are "Updated" or inputs/outputs for the process // --------------------------------------------------------------------- // NOTE: Cloud borne aerosols are not updated in this process but are included - // to create datastructures. + // to create data structures. + // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { @@ -191,6 +193,7 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); dry_atm_.pblh = get_field_in("pbl_height").get_view(); dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); // store fields converted to dry mmr from wet mmr in dry_atm_ dry_atm_.z_mid = buffer_.z_mid; @@ -204,9 +207,9 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { dry_atm_.w_updraft = buffer_.w_updraft; dry_atm_.z_surf = 0.0; // FIXME: for now + // Constituent fluxes at the surface (gasses and aerosols) [kg/m2/s] constituent_fluxes_ = get_field_in("constituent_fluxes").get_view(); - dry_atm_.p_del = get_field_in("pseudo_density").get_view(); // interstitial and cloudborne aerosol tracers of interest: mass (q) and // number (n) mixing ratios @@ -249,12 +252,13 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { //----------------------------------------------------------------- // Allocate memory //----------------------------------------------------------------- - + // Inverse of pseudo_density (1/Pa or or N/m2 or kg/m/s2) rpdel_ = view_2d("rpdel_", ncol_, nlev_); + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- - // preprocess_.initialize(constituent_fluxes_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); } // end initialize_impl() @@ -265,29 +269,39 @@ void MAMConstituentFluxes::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + // preprocess input -- needs a scan for the calculation of atm height - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + // policy haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] + // compute inverse of pseudo_density mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); - // ------------------------------------------------------------------- - // NOTES: The following code is an adaptation of cflx.F90 code in E3SM - // In EAMxx, all constituents are considered "wet" (or have wet mixing - // ratios), so we are *not* doing any wet to dry conversions here. We - // are simply updating the MAM4xx tracers using the "constituent - // fluxes" - // ------------------------------------------------------------------- + std::cout << "BEFORE:::" << wet_aero_.gas_mmr[4](1, 1) << std::endl; + // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - /////////dddd + // To construct state_q, we need fields from Prognostics (aerosols) // and Atmosphere (water species such as qv, qc etc.) @@ -298,10 +312,9 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); - // Construct state_q (interstitial) and qqcw (cloud borne) arrays - constexpr auto pver = mam4::ndrop::pver; + // Construct state_q (interstitial) array Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, pver), [&](int klev) { + Kokkos::TeamVectorRange(team, nlev_), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -312,16 +325,15 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - std::cout << "istart:" << mam4::utils::gasses_start_ind() - << std::endl; - // kg/m2/s to kg/kg/ + + // Factor to convert units from kg/m2/s to kg/kg Real unit_factor = dt * gravit * rpdel_(icol, klev); + // Update state with the constituent fluxes (modified // units) for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { - state_q_at_lev_col[icnst] = - state_q_at_lev_col[icnst] + + state_q_at_lev_col[icnst] += constituent_fluxes_(icol, icnst) * unit_factor; } mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, @@ -329,65 +341,8 @@ void MAMConstituentFluxes::run_impl(const double dt) { }); }); - /* ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) - tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver)*/ - - /*FORTRAN CODE: - What we need: - 1. ncol - 2. all tracers - 3. timestep (verify what is rztodt??) - 4. gravit, rpdel - 5. cflx - 6. Ensure dry/et-wet/dry isconsistent....comment on that!! - - - ncol = state%ncol - - !------------------------------------------------------- - ! Assume 'wet' mixing ratios in surface diffusion code. - - cnst_type_loc(:) = cnst_type(:) - call set_dry_to_wet(state, cnst_type_loc) - - !------------------------------------------------------- - ! Initialize ptend - - lq(:) = .TRUE. - call physics_ptend_init(ptend, state%psetcols, 'clubb_srf', lq=lq) - - !------------------------------------------------------- - ! Calculate tracer mixing ratio tendencies from cflx - - rztodt = 1._r8/ztodt - ptend%q(:ncol,:pver,:) = state%q(:ncol,:pver,:) - tmp1(:ncol) = ztodt * gravit * state%rpdel(:ncol,pver) - - do m = 2, pcnst - ptend%q(:ncol,pver,m) = ptend%q(:ncol,pver,m) + tmp1(:ncol) * - cam_in%cflx(:ncol,m) enddo - - ptend%q(:ncol,:pver,:) = (ptend%q(:ncol,:pver,:) - state%q(:ncol,:pver,:)) * - rztodt - - ! Convert tendencies of dry constituents to dry basis. - do m = 1,pcnst - if (cnst_type(m).eq.'dry') then - ptend%q(:ncol,:pver,m) = - ptend%q(:ncol,:pver,m)*state%pdel(:ncol,:pver)/state%pdeldry(:ncol,:pver) - endif - end do - - !------------------------------------------------------- - ! convert wet mmr back to dry before conservation check - ! avoid converting co2 tracers again - - cnst_type_loc(:) = cnst_type(:) - call co2_cycle_set_cnst_type(cnst_type_loc, 'wet') - call set_wet_to_dry(state, cnst_type_loc) - - */ -} + std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; +} // run_impl ends // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 02c4b879093..81c6ee52985 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -69,6 +69,47 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::DryAtmosphere &dry_atm) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + dry_atm_pre_ = dry_atm; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + } // operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + }; // Preprocess + + private: + // preprocessing scratch pads + Preprocess preprocess_; + }; // MAMConstituentFluxes } // namespace scream From 3496c7ddccec197485379e0d96cd70f7e34d110b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 06:53:57 -0700 Subject: [PATCH 36/65] Cleanup of some unused hpp file includes and some variables --- ...eamxx_mam_constituent_fluxes_interface.hpp | 1 + ...and_online_emissions_process_interface.cpp | 17 ++++++------- ...and_online_emissions_process_interface.hpp | 25 ++++++------------- .../single-process/mam/emissions/output.yaml | 2 +- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 81c6ee52985..448ae5464b8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -4,6 +4,7 @@ // For declaring contituent fluxes class derived from atm process // class #include + // For MAM4 aerosol configuration #include #include diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 01167845fd5..1fc3ddb4b68 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,4 +1,4 @@ -#include +//#include #include namespace scream { @@ -10,7 +10,7 @@ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { /* Anything that can be initialized without grid information can be - * initialized here. Like universal constants, mam wetscav options. + * initialized here. Like universal constants. */ } @@ -22,7 +22,8 @@ void MAMSrfOnlineEmiss::set_grids( grid_ = grids_manager->get_grid("Physics"); const auto &grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); @@ -32,16 +33,11 @@ void MAMSrfOnlineEmiss::set_grids( const FieldLayout scalar2d_pcnct = grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); - // -------------------------------------------------------------------------- - // These variables are "Required" or pure inputs for the process - // -------------------------------------------------------------------------- - // Temperature[K] at midpoints - add_field("T_mid", scalar3d_mid, K, grid_name); - // ------------------------------------------------------------- // These variables are "Computed" or outputs for the process // ------------------------------------------------------------- static constexpr Units m2(m * m, "m2"); + // Constituent fluxes of species in [kg/m2/s] add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); @@ -189,6 +185,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- // Output fields // --------------------------------------------------------------- + // Constituent fluxes of species in [kg/m2/s] constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); // --------------------------------------------------------------- @@ -227,7 +224,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, 1); // preprocess input -- needs a scan for the calculation of atm height Kokkos::parallel_for("preprocess", scan_policy, preprocess_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 1669125b9fb..b67b1de0bd7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -1,22 +1,19 @@ #ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP #define EAMXX_MAM_SRF_ONLINE_EMISS_HPP -#include -#include -#include - #include "share/grid/remap/abstract_remapper.hpp" #include "share/io/scorpio_input.hpp" + // For MAM4 aerosol configuration #include #include -#include + // For declaring surface and online emission class derived from atm process // class #include + // #include #include -#include namespace scream { @@ -30,22 +27,16 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; - // Wet and dry states of atmosphere - mam_coupling::WetAtmosphere wet_atm_; - mam_coupling::DryAtmosphere dry_atm_; - - // aerosol state variables - mam_coupling::AerosolState wet_aero_, dry_aero_; - // buffer for sotring temporary variables mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; + // Constituent fluxes of species in [kg/m2/s] view_2d constituent_fluxes_; - // work array to store fluxes after unit conversions to kg/m2/s + // Work array to store fluxes after unit conversions to kg/m2/s view_1d fluxes_in_mks_units_; // Unified atomic mass unit used for unit conversion (BAD constant) @@ -54,9 +45,6 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { public: using srfEmissFunc = mam_coupling::srfEmissFunctions; - template - using uview_2d = Unmanaged>; - // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -132,10 +120,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct srf_emiss_ { // species name std::string species_name; + // Data file name std::string data_file; + // Sector names in file std::vector sectors; + // Data structure for reading interpolation std::shared_ptr horizInterp_; std::shared_ptr dataReader_; diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml index b6e2d545d3c..433f7d58fa4 100644 --- a/components/eamxx/tests/single-process/mam/emissions/output.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -5,7 +5,7 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid + - constituent_fluxes output_control: Frequency: 1 frequency_units: nsteps From 4048ae83f28f9e32b19d195b8c055023a3496ebe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 07:49:58 -0700 Subject: [PATCH 37/65] Removes unused variables --- ...and_online_emissions_process_interface.cpp | 64 +++---------------- .../src/physics/mam/srf_emission_impl.hpp | 16 +---- 2 files changed, 13 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 1fc3ddb4b68..91a5b161dff 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -26,10 +26,8 @@ void MAMSrfOnlineEmiss::set_grids( nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; - FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); static constexpr int pcnst = mam4::aero_model::pcnst; - const FieldLayout scalar2d_pcnct = grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); @@ -149,7 +147,7 @@ void MAMSrfOnlineEmiss::set_grids( ispec_srf.data_out_, ispec_srf.dataReader_); } -} // set_grid +} // set_grid ends // ================================================================ // REQUEST_BUFFER_SIZE_IN_BYTES @@ -192,16 +190,16 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // Allocate memory for local and work arrays // --------------------------------------------------------------- - // work array to store fluxes after unit conversions to kg/m2/s + // Work array to store fluxes after unit conversions to kg/m2/s fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; - // Load the first month into srfEmiss_end. + // Load the first month into data_end. - // Note: At the first time step, the data will be moved into srfEmiss_beg, - // and srfEmiss_end will be reloaded from file with the new month. + // Note: At the first time step, the data will be moved into data_beg, + // and data_end will be reloaded from file with the new month. //-------------------------------------------------------------------- // Update surface emissions from file @@ -209,7 +207,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { for(srf_emiss_ &ispec_srf : srf_emiss_species_) { srfEmissFunc::update_srfEmiss_data_from_file( ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, - ispec_srf.data_end_); + ispec_srf.data_end_); // output } //----------------------------------------------------------------- @@ -242,12 +240,13 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { //-------------------------------------------------------------------- for(srf_emiss_ &ispec_srf : srf_emiss_species_) { - // Update srfEmissTimeState, note the addition of dt + // Update TimeState, note the addition of dt ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); // Update time state and if the month has changed, update the data. srfEmissFunc::update_srfEmiss_timestate( ispec_srf.dataReader_, ts, *ispec_srf.horizInterp_, + // output ispec_srf.timeState_, ispec_srf.data_start_, ispec_srf.data_end_); // Call the main srfEmiss routine to get interpolated aerosol forcings. @@ -280,51 +279,8 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // update flux in constituent_fluxes_ view Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); - } - - for(int i = 19; i < 30; ++i) { - std::cout << "BALLI:" << srf_emiss_species_[8].data_out_.emiss_sectors[0](i) - << ":" << i << ":" << constituent_fluxes_(i, 37) << std::endl; - } - - /* Rough notes: - - Here we should implement or port the chem_emissions subroutine in - chemistry.F90. Basically call two subroutines, aero_model_emissions - and set_srf_emissions. - - Here is the code: - - ! initialize chemistry constituent surface fluxes to zero - do m = 2,pcnst - n = map2chm(m) - if (n>0) cam_in%cflx(:,m) = 0._r8 - enddo - - ! aerosol emissions ... - call aero_model_emissions( state, & ! in - cam_in ) ! out - - ! prescribed emissions from file ... - - !----------------------------------------------------------------------- - ! ... Set surface emissions - !----------------------------------------------------------------------- - call set_srf_emissions( lchnk, ncol, sflx(:,:) ) - - do m = 1,pcnst - n = map2chm(m) - if ( n /= h2o_ndx .and. n > 0 ) then - cam_in%cflx(:ncol,m) = cam_in%cflx(:ncol,m) + sflx(:ncol,n) - call outfld( sflxnam(m), cam_in%cflx(:ncol,m), ncol,lchnk ) - endif - enddo - - - */ - - std::cout << "End of surface emissions run" << std::endl; -} + } // for loop for species +} // run_imple ends // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index b792c805b32..3fa458450a3 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -1,20 +1,9 @@ #ifndef SRF_EMISSION_IMPL_HPP #define SRF_EMISSION_IMPL_HPP -#include -#include -#include -#include -#include - -#include "physics/share/physics_constants.hpp" -#include "share/grid/remap/coarsening_remapper.hpp" #include "share/grid/remap/identity_remapper.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" -#include "share/io/scorpio_input.hpp" #include "share/io/scream_scorpio_interface.hpp" -#include "share/scream_types.hpp" -#include "share/util/scream_timing.hpp" namespace scream::mam_coupling { namespace { @@ -297,8 +286,9 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Read fields from the file for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { - auto aa = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], aa); + auto sector = + srfEmiss_horiz_interp.get_tgt_field(i).get_view(); + Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], sector); } Kokkos::fence(); From 0b9b81d2875b838cec329926e61910a3dcd379fe Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 11:55:58 -0700 Subject: [PATCH 38/65] Cleanup - removed unused codes and added kokkos reduction loop --- .../eamxx/src/physics/mam/srf_emission.hpp | 5 + .../src/physics/mam/srf_emission_impl.hpp | 143 +++--------------- 2 files changed, 25 insertions(+), 123 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 585656b0ec1..82a4b311382 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -8,6 +8,11 @@ namespace { template struct srfEmissFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + struct srfEmissTimeState { srfEmissTimeState() = default; // Whether the timestate has been initialized. diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 3fa458450a3..ef99879277e 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -8,71 +8,6 @@ namespace scream::mam_coupling { namespace { -template -template -std::shared_ptr -srfEmissFunctions::create_horiz_remapper( - const std::shared_ptr &model_grid, - const std::string &data_file, - const std::array §or_names, - const std::string &map_file) { - using namespace ShortFieldTagsNames; - - scorpio::register_file(data_file, scorpio::Read); - const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); - scorpio::release_file(data_file); - - // We could use model_grid directly if using same num levels, - // but since shallow clones are cheap, we may as well do it (less lines of - // code) - auto horiz_interp_tgt_grid = - model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); - - const int ncols_model = model_grid->get_num_global_dofs(); - std::shared_ptr remapper; - if(ncols_data == ncols_model) { - remapper = std::make_shared( - horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); - } else { - EKAT_REQUIRE_MSG(ncols_data <= ncols_model, - "Error! We do not allow to coarsen srfEmiss data to fit " - "the model. We only allow\n" - "srfEmiss data to be at the same or coarser resolution as " - "the model.\n"); - // We must have a valid map file - EKAT_REQUIRE_MSG( - map_file != "", - "ERROR: srfEmiss data is on a different grid than the model one,\n" - "but srfEmiss_remap_file is missing from srfEmiss parameter " - "list."); - - remapper = - std::make_shared(horiz_interp_tgt_grid, map_file); - } - - remapper->registration_begins(); - - const auto tgt_grid = remapper->get_tgt_grid(); - - const auto layout_2d = tgt_grid->get_2d_scalar_layout(); - const auto nondim = ekat::units::Units::nondimensional(); - - std::vector emiss_sectors; - - for(int icomp = 0; icomp < numSectors; ++icomp) { - auto comp_name = sector_names[icomp]; - // set and allocate fields - Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); - f.allocate_view(); - emiss_sectors.push_back(f); - remapper->register_field_from_tgt(f); - } - - remapper->registration_ends(); - - return remapper; -} // create_horiz_remapper - template std::shared_ptr srfEmissFunctions::create_horiz_remapper( @@ -186,41 +121,29 @@ void srfEmissFunctions::perform_time_interpolation( std::to_string(t_beg) + "\n delta_t: " + std::to_string(delta_t) + "\n"); - for(int icol = 0; icol < data_beg.data.ncols; ++icol) { - Real accum = 0; - for(int i = 0; i < data_beg.data.nsectors; ++i) { - accum += - linear_interp(data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), delta_t_fraction); - if(icol == 19) std::cout << "accum:" << accum << std::endl; - } - data_out.emiss_sectors[0](icol) = accum; - } - - /*const auto policy = ESU::get_default_team_policy(data_beg.data.ncols, 1); + const int nsectors = data_beg.data.nsectors; + const int ncols = data_beg.data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nsectors); Kokkos::parallel_for( - "srfEmiss_time_interp_loop", policy, - KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int icol = team.league_rank(); - double result; - - Kokkos::parallel_reduce("Loop1", N, KOKKOS_LAMBDA (const int& i, double& - lsum) { lsum += 1.0*i; - }, result); - - Kokkos::parallel_reduce("srfEmiss_reduction_loop", N, KOKKOS_LAMBDA - (const int& i, double& lsum) { for(int i = 0; i < data_beg.data.nsectors; ++i) - { Kokkos::single(Kokkos::PerTeam(team), [&] { accum[icol] = accum[icol] + - linear_interp(data_beg.data.emiss_sectors[i](icol), - data_end.data.emiss_sectors[i](icol), - delta_t_fraction); - - }); - } - data_out.emiss_sectors[0](icol) = accum[icol]; + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int icol = team.league_rank(); // column index + Real accum = 0; + // Parallel reduction over sectors + // FIXME: Do we need to use Kokkos::Single for each team here??? + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, nsectors), + [&](const int i, Real &update) { + const auto beg = data_beg.data.emiss_sectors[i](icol); + const auto end = data_end.data.emiss_sectors[i](icol); + update += linear_interp(beg, end, delta_t_fraction); + }, + accum); + // Assign the accumulated value to the output + data_out.emiss_sectors[0](icol) = accum; }); - Kokkos::fence();*/ } // perform_time_interpolation template @@ -332,32 +255,6 @@ void srfEmissFunctions::update_srfEmiss_timestate( } // END updata_srfEmiss_timestate -template -template -void srfEmissFunctions::init_srf_emiss_objects( - const int ncol, const std::shared_ptr &grid, - const std::string &data_file, - const std::array §ors, - const std::string &srf_map_file, - // output - std::shared_ptr &SrfEmissHorizInterp, - srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, - srfEmissOutput &SrfEmissData_out, - std::shared_ptr &SrfEmissDataReader) { - // Init horizontal remap - SrfEmissHorizInterp = - create_horiz_remapper(grid, data_file, sectors, srf_map_file); - - // Initialize the size of start/end/out data structures - SrfEmissData_start = srfEmissInput(ncol, numSectors); - SrfEmissData_end = srfEmissInput(ncol, numSectors); - SrfEmissData_out.init(ncol, 1, true); - - // Create reader (an AtmosphereInput object) - SrfEmissDataReader = - create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); -} // init_srf_emiss_objects - template void srfEmissFunctions::init_srf_emiss_objects( const int ncol, const std::shared_ptr &grid, From 0c258d47649c18ffc4e30b8c2ab00c6f5b4744aa Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 2 Aug 2024 12:25:53 -0700 Subject: [PATCH 39/65] Adds output fileds to yaml and also change the location of emissions files --- .../src/physics/mam/srf_emission_impl.hpp | 3 +- .../mam/constituent_fluxes/output.yaml | 33 ++++++++++++++++++- .../mam/emissions/CMakeLists.txt | 16 +++++++++ .../single-process/mam/emissions/input.yaml | 18 +++++----- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index ef99879277e..5191966f588 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -144,6 +144,7 @@ void srfEmissFunctions::perform_time_interpolation( // Assign the accumulated value to the output data_out.emiss_sectors[0](icol) = accum; }); + Kokkos::fence(); } // perform_time_interpolation template @@ -282,4 +283,4 @@ void srfEmissFunctions::init_srf_emiss_objects( } // namespace } // namespace scream::mam_coupling -#endif // SRF_EMISSION_IMPL_HPP \ No newline at end of file +#endif // SRF_EMISSION_IMPL_HPP diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml index e51b96eb5ce..b237056b5cc 100644 --- a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml @@ -5,7 +5,38 @@ Averaging Type: Instant Fields: Physics: Field Names: - - T_mid + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + output_control: Frequency: 1 frequency_units: nsteps diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index b57adcb60ef..3d64bc5b2b2 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -25,6 +25,22 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 2e63314ebb1..06be9ec8a88 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,15 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/test_DECK_ne30/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From c29bab538696ea6fedbfac421d55a1049f4b6029 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 3 Aug 2024 13:42:55 -0700 Subject: [PATCH 40/65] Some GPU fixes but needs some cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 5c5129d1b7c..564fa9465fa 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -266,6 +266,7 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -286,7 +287,8 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::fence(); // policy - haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] @@ -296,7 +298,17 @@ void MAMConstituentFluxes::run_impl(const double dt) { nlev_, // output rpdel_); - std::cout << "BEFORE:::" << wet_aero_.gas_mmr[4](1, 1) << std::endl; + auto wet_aero = wet_aero_; + auto dry_atm = dry_atm_; + auto nlev = nlev_; + auto rpdel = rpdel_; + auto constituent_fluxes = constituent_fluxes_; + + + //auto aa = wet_aero.gas_mmr[4](1, 1); + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); + + printf("BEFORE:::%e\n",host_view(0,0)); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { @@ -307,14 +319,15 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get prognostics mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero_, icol); + mam_coupling::aerosols_for_column(wet_aero, icol); + //printf("in-loop-wetaero:,%e,%e\n", wet_aero.gas_mmr[4](0,0), progs_at_col.q_gas[4](0)); // get atmospheric quantities - haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm_, icol); + haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); // Construct state_q (interstitial) array Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev_), [&](int klev) { + Kokkos::TeamVectorRange(team, nlev), [&](int klev) { Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; // get state_q at a grid cell (col,lev) @@ -323,25 +336,35 @@ void MAMConstituentFluxes::run_impl(const double dt) { mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_lev_col, klev); + // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); // Factor to convert units from kg/m2/s to kg/kg - Real unit_factor = dt * gravit * rpdel_(icol, klev); + Real unit_factor = dt * gravit * rpdel(icol, klev); // Update state with the constituent fluxes (modified // units) + printf("in-loop-klev:,%e\n", progs_at_col.q_gas[4](icol)); for(int icnst = istart; icnst < mam4::aero_model::pcnst; ++icnst) { state_q_at_lev_col[icnst] += - constituent_fluxes_(icol, icnst) * unit_factor; + constituent_fluxes(icol, icnst) * unit_factor; + //printf("in-loop-update:,%e,%e, %i\n", progs_at_col.q_gas[icnst](icol), state_q_at_lev_col[icnst], icnst); + printf("INLOOP:::%e, %i\n",state_q_at_lev_col[icnst], icnst); } mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, progs_at_col, klev); + }); }); - - std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; + auto aa1 = wet_aero.gas_mmr; + auto bb1 = aa1[4]; + auto host_view1 = Kokkos::create_mirror_view(bb1); + auto cc1 = host_view1(1,1); + printf("AFTER:::%e\n",cc1); + printf("Scientific notation:");//, wet_aero.gas_mmr[4](1, 1)); + //std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; } // run_impl ends // ============================================================================= From e4b0aa27768e0e760e7c8c8dc47ecb32c093581e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 3 Aug 2024 23:17:38 -0700 Subject: [PATCH 41/65] Reworked logic to add fluxes only at the surface level-needs cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 90 ++++++------------- ...eamxx_mam_constituent_fluxes_interface.hpp | 3 - 2 files changed, 29 insertions(+), 64 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 564fa9465fa..7fc360d92c8 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -249,12 +249,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(gas_mmr_field_name).get_view(); } - //----------------------------------------------------------------- - // Allocate memory - //----------------------------------------------------------------- - // Inverse of pseudo_density (1/Pa or or N/m2 or kg/m/s2) - rpdel_ = view_2d("rpdel_", ncol_, nlev_); - //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- @@ -266,7 +260,6 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); @@ -287,84 +280,59 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::fence(); // policy - //haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); + // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] - // compute inverse of pseudo_density - mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, - nlev_, - // output - rpdel_); - auto wet_aero = wet_aero_; - auto dry_atm = dry_atm_; - auto nlev = nlev_; - auto rpdel = rpdel_; + auto wet_aero = wet_aero_; + auto dry_atm = dry_atm_; + const int surface_lev = nlev_ - 1; auto constituent_fluxes = constituent_fluxes_; + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); - - //auto aa = wet_aero.gas_mmr[4](1, 1); + // auto aa = wet_aero.gas_mmr[4](1, 1); auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); - printf("BEFORE:::%e\n",host_view(0,0)); + for(int icnst = 0; icnst < 5; ++icnst) { + printf("BEFORE:::%e,%e\n", host_view(icnst, 71), + wet_aero_.gas_mmr[4](icnst, 71)); + } + + const auto policy_pcnst = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol_, mam4::aero_model::pcnst); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - // To construct state_q, we need fields from Prognostics (aerosols) - // and Atmosphere (water species such as qv, qc etc.) + EKAT_REQUIRE_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must not be zero!"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - // get prognostics mam4::Prognostics progs_at_col = mam_coupling::aerosols_for_column(wet_aero, icol); - //printf("in-loop-wetaero:,%e,%e\n", wet_aero.gas_mmr[4](0,0), progs_at_col.q_gas[4](0)); // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); + Real state_q_at_surf_lev[mam4::aero_model::pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); - // Construct state_q (interstitial) array - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nlev), [&](int klev) { - Real state_q_at_lev_col[mam4::aero_model::pcnst] = {}; - - // get state_q at a grid cell (col,lev) - // NOTE: The order of species in state_q_at_lev_col - // is the same as in E3SM state%q array - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_lev_col, klev); - - - // get the start index for gas species in the state_q array - int istart = mam4::utils::gasses_start_ind(); + const Real unit_factor = dt * gravit * rpdel; - // Factor to convert units from kg/m2/s to kg/kg - Real unit_factor = dt * gravit * rpdel(icol, klev); - - // Update state with the constituent fluxes (modified - // units) - printf("in-loop-klev:,%e\n", progs_at_col.q_gas[4](icol)); - for(int icnst = istart; icnst < mam4::aero_model::pcnst; - ++icnst) { - state_q_at_lev_col[icnst] += + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, istart, mam4::aero_model::pcnst), + [&](int icnst) { + state_q_at_surf_lev[icnst] += constituent_fluxes(icol, icnst) * unit_factor; - //printf("in-loop-update:,%e,%e, %i\n", progs_at_col.q_gas[icnst](icol), state_q_at_lev_col[icnst], icnst); - printf("INLOOP:::%e, %i\n",state_q_at_lev_col[icnst], icnst); - } - mam4::utils::inject_stateq_to_prognostics(state_q_at_lev_col, - progs_at_col, klev); - }); + mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, + progs_at_col, surface_lev); }); - auto aa1 = wet_aero.gas_mmr; - auto bb1 = aa1[4]; - auto host_view1 = Kokkos::create_mirror_view(bb1); - auto cc1 = host_view1(1,1); - printf("AFTER:::%e\n",cc1); - printf("Scientific notation:");//, wet_aero.gas_mmr[4](1, 1)); - //std::cout << wet_aero_.gas_mmr[4](1, 1) << std::endl; } // run_impl ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 448ae5464b8..f5d153360b9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -15,7 +15,6 @@ namespace scream { // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMConstituentFluxes final : public scream::AtmosphereProcess { using KT = ekat::KokkosTypes; - using view_2d = Field::view_dev_t; using const_view_2d = Field::view_dev_t; // number of horizontal columns @@ -36,8 +35,6 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { const_view_2d constituent_fluxes_; - view_2d rpdel_; // Inverse of pdel_ or pseudo_density - public: // Constructor MAMConstituentFluxes(const ekat::Comm &comm, From 4c5f58c63b3fecd2059bf5e0535b6e60e4646bb0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 4 Aug 2024 00:06:51 -0700 Subject: [PATCH 42/65] Fixes pm-gpu build and run-needs cleanup --- ...eamxx_mam_constituent_fluxes_interface.cpp | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 7fc360d92c8..9fc712ba46f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -285,10 +285,11 @@ void MAMConstituentFluxes::run_impl(const double dt) { using C = physics::Constants; static constexpr auto gravit = C::gravit; // Gravity [m/s2] - + const int ncol = ncol_; auto wet_aero = wet_aero_; auto dry_atm = dry_atm_; const int surface_lev = nlev_ - 1; + static constexpr int pcnst = mam4::aero_model::pcnst; auto constituent_fluxes = constituent_fluxes_; // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); @@ -297,20 +298,19 @@ void MAMConstituentFluxes::run_impl(const double dt) { auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); for(int icnst = 0; icnst < 5; ++icnst) { - printf("BEFORE:::%e,%e\n", host_view(icnst, 71), - wet_aero_.gas_mmr[4](icnst, 71)); - } + printf("BEFORE:::%e\n", host_view(icnst, 71)); + } const auto policy_pcnst = ekat::ExeSpaceUtils::get_default_team_policy( - ncol_, mam4::aero_model::pcnst); + ncol, pcnst); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { const int icol = team.league_rank(); - EKAT_REQUIRE_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must not be zero!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must not be zero!\n"); const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); mam4::Prognostics progs_at_col = @@ -318,21 +318,24 @@ void MAMConstituentFluxes::run_impl(const double dt) { // get atmospheric quantities haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); - Real state_q_at_surf_lev[mam4::aero_model::pcnst] = {}; + Real state_q_at_surf_lev[pcnst] = {}; mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); const Real unit_factor = dt * gravit * rpdel; - + auto pcnst1 = pcnst; Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, mam4::aero_model::pcnst), + Kokkos::TeamVectorRange(team, istart, pcnst1), [&](int icnst) { state_q_at_surf_lev[icnst] += constituent_fluxes(icol, icnst) * unit_factor; }); mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, - progs_at_col, surface_lev); + progs_at_col, surface_lev); }); + for(int icnst = 0; icnst < 5; ++icnst) { + printf("AFTER:::%e\n", host_view(icnst, 71)); + } } // run_impl ends // ============================================================================= From 209ba3db2733081f2125559c56359f6a989e8d5d Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 5 Aug 2024 19:13:22 -0700 Subject: [PATCH 43/65] Moved Kokkos - loop outside of interface and verified that gas and aerosols mmrs are modified --- ...eamxx_mam_constituent_fluxes_functions.hpp | 105 ++++++++++++++++++ ...eamxx_mam_constituent_fluxes_interface.cpp | 72 +++--------- ...eamxx_mam_constituent_fluxes_interface.hpp | 2 + 3 files changed, 122 insertions(+), 57 deletions(-) create mode 100644 components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp new file mode 100644 index 00000000000..9059cf5ccbc --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -0,0 +1,105 @@ +#ifndef EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP +#define EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP + +#include +#include + +namespace scream { + +namespace { + +void update_gas_aerosols_using_constituents( + const int ncol, const int nlev, const double dt, + const mam_coupling::DryAtmosphere &dry_atm, + const MAMConstituentFluxes::const_view_2d &constituent_fluxes, + // output + const mam_coupling::AerosolState &wet_aero) { + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr int pcnst = mam4::aero_model::pcnst; + + // Declare local variables + const int ncol_loc = ncol; + auto wet_aero_loc = wet_aero; + auto dry_atm_loc = dry_atm; + const int surface_lev = nlev - 1; + auto constituent_fluxes_loc = constituent_fluxes; + + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); + + // number of constituents to update + const int nconstituents = pcnst - istart; + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, surface_lev), icnst + 9); + } + + // Create a policy to loop over columns annd number of constituents + // to update + // const auto policy = + // ekat::ExeSpaceUtils::get_default_team_policy(ncol, + // nconstituents); + const auto policy = ekat::ExeSpaceUtils:: + get_default_team_policy(1, nconstituents); + + // Loop through all columns to update tracer mixing rations + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + //---------------------------------------------------------------------- + // To form EAM like state%q array, we need prognostics (gas and aerosol + // mmrs) atmosphere (qv, qc, nc, ni, etc.) + //---------------------------------------------------------------------- + + // Get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(wet_aero, // output + icol); // input + + // Get atmospheric quantities + const haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atm, // output + icol); // input + + // Form state%q like array + Real state_q_at_surf_lev[pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, // input + state_q_at_surf_lev, // output + surface_lev); // input + + // Compute the units conversion factor (kg/m2/s to kg/kg) + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must be non-zero!\n"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); + const Real unit_factor = dt * gravit * rpdel; + + // Loop for + auto pcnst_loc = pcnst; + for(int icnst = 9; icnst < 15; ++icnst) { + printf("BEFORE-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], + progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); + } + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, istart, pcnst_loc), [&](int icnst) { + state_q_at_surf_lev[icnst] += + constituent_fluxes(icol, icnst) * unit_factor; + }); // pcsnt loop + mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, // input + progs_at_col, // output + surface_lev); // input + + for(int icnst = 9; icnst < 15; ++icnst) { + printf("After-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], + progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); + } + }); // icol loop +} + +} // namespace +} // namespace scream + +#endif // ifdef EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 9fc712ba46f..1d1ffca2928 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -1,3 +1,4 @@ +#include #include namespace scream { @@ -260,8 +261,12 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { + // const auto scan_policy = ekat::ExeSpaceUtils< + // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, + // nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in @@ -279,63 +284,16 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // policy - // haero::ThreadTeamPolicy team_policy(ncol_, Kokkos::AUTO); - haero::ThreadTeamPolicy team_policy(1, Kokkos::AUTO); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view1 = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + printf("END:::%e, %i\n", host_view1(0, 71), icnst + 9); + } - using C = physics::Constants; - static constexpr auto gravit = C::gravit; // Gravity [m/s2] - const int ncol = ncol_; - auto wet_aero = wet_aero_; - auto dry_atm = dry_atm_; - const int surface_lev = nlev_ - 1; - static constexpr int pcnst = mam4::aero_model::pcnst; - auto constituent_fluxes = constituent_fluxes_; - // get the start index for gas species in the state_q array - int istart = mam4::utils::gasses_start_ind(); - - // auto aa = wet_aero.gas_mmr[4](1, 1); - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[4]); - - for(int icnst = 0; icnst < 5; ++icnst) { - printf("BEFORE:::%e\n", host_view(icnst, 71)); - } - - const auto policy_pcnst = - ekat::ExeSpaceUtils::get_default_team_policy( - ncol, pcnst); - // Loop through all columns to update tracer mixing rations - Kokkos::parallel_for( - policy_pcnst, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must not be zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, icol); - - // get atmospheric quantities - haero::Atmosphere haero_atm = atmosphere_for_column(dry_atm, icol); - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, state_q_at_surf_lev, surface_lev); - - const Real unit_factor = dt * gravit * rpdel; - auto pcnst1 = pcnst; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, pcnst1), - [&](int icnst) { - state_q_at_surf_lev[icnst] += - constituent_fluxes(icol, icnst) * unit_factor; - }); - mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, - progs_at_col, surface_lev); - }); - for(int icnst = 0; icnst < 5; ++icnst) { - printf("AFTER:::%e\n", host_view(icnst, 71)); - } } // run_impl ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index f5d153360b9..7e77fccf2af 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -14,9 +14,11 @@ namespace scream { // The process responsible for applying MAM4 constituent fluxes. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMConstituentFluxes final : public scream::AtmosphereProcess { + public: using KT = ekat::KokkosTypes; using const_view_2d = Field::view_dev_t; + private: // number of horizontal columns int ncol_, nlev_; From d4c9e2ffbcedb5fedb26911d578ac2b403dfd062 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 9 Aug 2024 08:06:45 -0700 Subject: [PATCH 44/65] Fixes some GPU errors and warnings --- ...eamxx_mam_constituent_fluxes_functions.hpp | 107 +++++++----------- ...eamxx_mam_constituent_fluxes_interface.cpp | 78 +++++++------ 2 files changed, 89 insertions(+), 96 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index 9059cf5ccbc..c53c09eacbd 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -19,7 +19,6 @@ void update_gas_aerosols_using_constituents( static constexpr int pcnst = mam4::aero_model::pcnst; // Declare local variables - const int ncol_loc = ncol; auto wet_aero_loc = wet_aero; auto dry_atm_loc = dry_atm; const int surface_lev = nlev - 1; @@ -28,75 +27,57 @@ void update_gas_aerosols_using_constituents( // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - // number of constituents to update + // number of constituents to update (currently updating only MAM4xx constituents) const int nconstituents = pcnst - istart; - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, surface_lev), icnst + 9); - } - // Create a policy to loop over columns annd number of constituents // to update - // const auto policy = - // ekat::ExeSpaceUtils::get_default_team_policy(ncol, - // nconstituents); - const auto policy = ekat::ExeSpaceUtils:: - get_default_team_policy(1, nconstituents); + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol, + nconstituents); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - //---------------------------------------------------------------------- - // To form EAM like state%q array, we need prognostics (gas and aerosol - // mmrs) atmosphere (qv, qc, nc, ni, etc.) - //---------------------------------------------------------------------- - - // Get prognostics - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, // output - icol); // input - - // Get atmospheric quantities - const haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atm, // output - icol); // input - - // Form state%q like array - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, // input - state_q_at_surf_lev, // output - surface_lev); // input - - // Compute the units conversion factor (kg/m2/s to kg/kg) - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must be non-zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - const Real unit_factor = dt * gravit * rpdel; - - // Loop for - auto pcnst_loc = pcnst; - for(int icnst = 9; icnst < 15; ++icnst) { - printf("BEFORE-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], - progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); - } - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, istart, pcnst_loc), [&](int icnst) { - state_q_at_surf_lev[icnst] += - constituent_fluxes(icol, icnst) * unit_factor; - }); // pcsnt loop - mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, // input - progs_at_col, // output - surface_lev); // input - - for(int icnst = 9; icnst < 15; ++icnst) { - printf("After-state:%e,%e,%i, %i\n", state_q_at_surf_lev[icnst], - progs_at_col.q_gas[icnst - 9](surface_lev), icnst, icol); - } - }); // icol loop + policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + //---------------------------------------------------------------------- + // To form EAM like state%q array, we need prognostics (gas and aerosol + // mmrs) atmosphere (qv, qc, nc, ni, etc.) + //---------------------------------------------------------------------- + + // Get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(wet_aero, // output + icol); // input + // Get atmospheric quantities + const haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atm, // output + icol); // input + + // Form state%q like array + Real state_q_at_surf_lev[pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, // input + state_q_at_surf_lev, // output + surface_lev); // input + + // Compute the units conversion factor (kg/m2/s to kg/kg) + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must be non-zero!\n"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); + const Real unit_factor = dt * gravit * rpdel; + + // Update state vector with constituent fluxes + for (int icnst = istart; icnst #include #include namespace scream { @@ -260,39 +261,50 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // ================================================================ // RUN_IMPL // ================================================================ -void MAMConstituentFluxes::run_impl(const double dt) { - // const auto scan_policy = ekat::ExeSpaceUtils< - // KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, - // nlev_); - - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(1, nlev_); - - // ------------------------------------------------------------------- - // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in - // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". - // We are converting wet atm to dry atm. Since we do not use or update - // any of the water constituents (qc, qv, qi etc.), we should be okay - // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. - // ------------------------------------------------------------------- - - // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, - constituent_fluxes_, - // output - wet_aero_); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view1 = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - printf("END:::%e, %i\n", host_view1(0, 71), icnst + 9); - } + void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, + nlev_); + + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + + // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ etc. + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } + auto start = std::chrono::steady_clock::now(); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + auto stop = std::chrono::steady_clock::now(); + auto duration = (stop - start); + + + // To get the value of duration use the count() + // member function on the duration object + printf("TIME:%e\n", std::chrono::duration(duration).count()); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } } // run_impl ends From 6f1e5d72d5f9e2135f18cd01e5042bc2d1e4bfd4 Mon Sep 17 00:00:00 2001 From: James Overfelt Date: Sat, 10 Aug 2024 07:51:55 -0600 Subject: [PATCH 45/65] Fix some compile warnings that are really errors. Calling host from device is not allowed and will fail at runtime. Also there were a couple of warnings about the implicit capture of the this pointer that turned out to cause runtime errors. --- ...and_online_emissions_process_interface.cpp | 36 ++++++------------- ...and_online_emissions_process_interface.hpp | 16 --------- .../eamxx/src/physics/mam/mam_coupling.hpp | 2 +- .../eamxx/src/physics/mam/srf_emission.hpp | 27 +++++++------- .../src/physics/mam/srf_emission_impl.hpp | 21 +++++------ 5 files changed, 35 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 91a5b161dff..09c173468a2 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -191,7 +191,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // --------------------------------------------------------------- // Work array to store fluxes after unit conversions to kg/m2/s - fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units_", ncol_); + fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units", ncol_); // Current month ( 0-based) const int curr_month = timestamp().get_month() - 1; @@ -221,16 +221,9 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, 1); - // preprocess input -- needs a scan for the calculation of atm height - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - // policy to loop over columns only - const auto ncol_only_policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol_, 1); + // Zero output + Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation auto ts = timestamp() + dt; @@ -261,25 +254,18 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { const int species_index = spcIndex_in_pcnst_.at(ispec_srf.species_name); // modify units from molecules/cm2/s to kg/m2/s + auto fluxes_in_mks_units = this->fluxes_in_mks_units_; + auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units - Kokkos::parallel_for( - ncol_only_policy, - KOKKOS_LAMBDA(const MAMSrfOnlineEmiss::KT::MemberType &team) { - const int icol = team.league_rank(); - fluxes_in_mks_units_(icol) = - ispec_srf.data_out_.emiss_sectors[0](icol) * mfactor; - }); - - // Get subview - auto flux_1d_view = - Kokkos::subview(constituent_fluxes_, Kokkos::ALL(), - spcIndex_in_pcnst_.at(ispec_srf.species_name)); - - // update flux in constituent_fluxes_ view - Kokkos::deep_copy(flux_1d_view, fluxes_in_mks_units_); + Kokkos::parallel_for("fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = ispec_srf.data_out_.emiss_sectors(0,icol) * mfactor; + constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); + }); + } // for loop for species + Kokkos::fence(); } // run_imple ends // ============================================================================= diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index b67b1de0bd7..031fb62d8b7 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -83,24 +83,8 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { void initialize(const view_2d &constituent_fluxes) { constituent_fluxes_pre_ = constituent_fluxes; } - - KOKKOS_INLINE_FUNCTION - void operator()( - const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index - // zero-out the constituent surface fluxes for all gas and aerosol - // species. - for(auto icnst = mam4::utils::gasses_start_ind(); - icnst < mam4::aero_model::pcnst; ++icnst) { - constituent_fluxes_pre_(i, icnst) = 0; - } - team.team_barrier(); - - } // operator() - // local variables for preprocess struct view_2d constituent_fluxes_pre_; - }; // MAMSrfOnlineEmiss::Preprocess private: diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 09d64d6028d..a3a5f746aa3 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -777,7 +777,7 @@ void compute_wet_mixing_ratios(const Team& team, } // Computes the reciprocal of pseudo density for a column -KOKKOS_INLINE_FUNCTION +inline void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, const_view_2d pdel, const int nlev, diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 82a4b311382..886d9d00609 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -30,26 +30,23 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_, const int nsectors_) { - init(ncol_, nsectors_, true); + srfEmissData(const int ncol_, const int nsectors_) : + ncols(ncol_), + nsectors(nsectors_) + { + init(ncols, nsectors, true); } - void init(const int ncol_, const int nsectors_, const bool allocate) { - ncols = ncol_; - nsectors = nsectors_; - - if(allocate) { - for(int i = 0; i < nsectors; ++i) { - auto sector = view_1d("", ncols); - emiss_sectors.push_back(sector); - } - } + void init(const int ncol, const int nsector, const bool allocate) { + ncols = ncol; + nsectors = nsector; + if(allocate) + emiss_sectors = view_2d("AllSectors", nsectors, ncols); } // srfEmissData init // Basic spatial dimensions of the data int ncols, nsectors; - std::vector emiss_sectors; - + view_2d emiss_sectors; }; // srfEmissData struct srfEmissInput { @@ -141,4 +138,4 @@ struct srfEmissFunctions { } // namespace scream::mam_coupling #endif // SRF_EMISSION_HPP -#include "srf_emission_impl.hpp" \ No newline at end of file +#include "srf_emission_impl.hpp" diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 5191966f588..35a2b90deeb 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -55,14 +55,14 @@ srfEmissFunctions::create_horiz_remapper( const auto layout_2d = tgt_grid->get_2d_scalar_layout(); const auto nondim = ekat::units::Units::nondimensional(); - std::vector emiss_sectors; + std::vector field_emiss_sectors; for(int icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); f.allocate_view(); - emiss_sectors.push_back(f); + field_emiss_sectors.push_back(f); remapper->register_field_from_tgt(f); } @@ -76,18 +76,18 @@ std::shared_ptr srfEmissFunctions::create_srfEmiss_data_reader( const std::shared_ptr &horiz_remapper, const std::string &srfEmiss_data_file) { - std::vector emiss_sectors; + std::vector field_emiss_sectors; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { - emiss_sectors.push_back(horiz_remapper->get_src_field(i)); + field_emiss_sectors.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); return std::make_shared(srfEmiss_data_file, io_grid, - emiss_sectors, true); + field_emiss_sectors, true); } // create_srfEmiss_data_reader template template -KOKKOS_INLINE_FUNCTION ScalarX srfEmissFunctions::linear_interp( +ScalarX srfEmissFunctions::linear_interp( const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { return (1 - t) * x0 + t * x1; } // linear_interp @@ -136,13 +136,13 @@ void srfEmissFunctions::perform_time_interpolation( Kokkos::parallel_reduce( Kokkos::TeamThreadRange(team, nsectors), [&](const int i, Real &update) { - const auto beg = data_beg.data.emiss_sectors[i](icol); - const auto end = data_end.data.emiss_sectors[i](icol); + const auto beg = data_beg.data.emiss_sectors(i, icol); + const auto end = data_end.data.emiss_sectors(i, icol); update += linear_interp(beg, end, delta_t_fraction); }, accum); // Assign the accumulated value to the output - data_out.emiss_sectors[0](icol) = accum; + data_out.emiss_sectors(0, icol) = accum; }); Kokkos::fence(); } // perform_time_interpolation @@ -212,7 +212,8 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - Kokkos::deep_copy(srfEmiss_input.data.emiss_sectors[i], sector); + const auto emiss = Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); + Kokkos::deep_copy(emiss, sector); } Kokkos::fence(); From 9c2fb6545c13b86ba1583be90bc36f71f13efd68 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 11:54:38 -0700 Subject: [PATCH 46/65] Fixes a warning for a size type for a string vector --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 35a2b90deeb..7a677bbdca3 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,7 +57,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(int icomp = 0; icomp < sector_names.size(); ++icomp) { + for(std::vector::size_type icomp = 0; icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From c45a86a5d267d1a460990ca05b38522fa4662006 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 12:02:47 -0700 Subject: [PATCH 47/65] Clang format --- ...eamxx_mam_constituent_fluxes_functions.hpp | 87 +++++++++--------- ...eamxx_mam_constituent_fluxes_interface.cpp | 88 +++++++++---------- ...and_online_emissions_process_interface.cpp | 15 ++-- .../eamxx/src/physics/mam/srf_emission.hpp | 11 +-- .../src/physics/mam/srf_emission_impl.hpp | 11 ++- 5 files changed, 106 insertions(+), 106 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index c53c09eacbd..e7ffb6376c3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -27,57 +27,56 @@ void update_gas_aerosols_using_constituents( // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); - // number of constituents to update (currently updating only MAM4xx constituents) + // number of constituents to update (currently updating only MAM4xx + // constituents) const int nconstituents = pcnst - istart; // Create a policy to loop over columns annd number of constituents // to update - const auto policy = - ekat::ExeSpaceUtils::get_default_team_policy(ncol, - nconstituents); + const auto policy = ekat::ExeSpaceUtils:: + get_default_team_policy(ncol, nconstituents); // Loop through all columns to update tracer mixing rations Kokkos::parallel_for( - policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - - //---------------------------------------------------------------------- - // To form EAM like state%q array, we need prognostics (gas and aerosol - // mmrs) atmosphere (qv, qc, nc, ni, etc.) - //---------------------------------------------------------------------- - - // Get prognostics - mam4::Prognostics progs_at_col = - mam_coupling::aerosols_for_column(wet_aero, // output - icol); // input - // Get atmospheric quantities - const haero::Atmosphere haero_atm = - atmosphere_for_column(dry_atm, // output - icol); // input - - // Form state%q like array - Real state_q_at_surf_lev[pcnst] = {}; - mam4::utils::extract_stateq_from_prognostics( - progs_at_col, haero_atm, // input - state_q_at_surf_lev, // output - surface_lev); // input - - // Compute the units conversion factor (kg/m2/s to kg/kg) - EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, - "Error! dry_atm.pdel must be non-zero!\n"); - const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); - const Real unit_factor = dt * gravit * rpdel; - - // Update state vector with constituent fluxes - for (int icnst = istart; icnst::get_thread_range_parallel_scan_team_policy(ncol_, - nlev_); - - // ------------------------------------------------------------------- - // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in - // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". - // We are converting wet atm to dry atm. Since we do not use or update - // any of the water constituents (qc, qv, qi etc.), we should be okay - // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. - // ------------------------------------------------------------------- - - // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ etc. - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - auto start = std::chrono::steady_clock::now(); - update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, - constituent_fluxes_, - // output - wet_aero_); - auto stop = std::chrono::steady_clock::now(); - auto duration = (stop - start); - - - // To get the value of duration use the count() - // member function on the duration object - printf("TIME:%e\n", std::chrono::duration(duration).count()); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } +void MAMConstituentFluxes::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), so we are *not* doing any wet to dry conversions in the + // "preprocess" . We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are build following HAERO data structures. + // ------------------------------------------------------------------- + + // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ + // etc. + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } + auto start = std::chrono::steady_clock::now(); + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); + auto stop = std::chrono::steady_clock::now(); + auto duration = (stop - start); + + // To get the value of duration use the count() + // member function on the duration object + printf("TIME:%e\n", + std::chrono::duration(duration).count()); + + for(int icnst = 0; icnst < 6; ++icnst) { + auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); + Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); + printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); + } } // run_impl ends diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 09c173468a2..a4ac4778982 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -221,8 +221,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - - // Zero output + // Zero output Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation @@ -255,14 +254,16 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // modify units from molecules/cm2/s to kg/m2/s auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - auto constituent_fluxes = this->constituent_fluxes_; + auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; // Parallel loop over all the columns to update units - Kokkos::parallel_for("fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - fluxes_in_mks_units(icol) = ispec_srf.data_out_.emiss_sectors(0,icol) * mfactor; - constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); - }); + Kokkos::parallel_for( + "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = + ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; + constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); + }); } // for loop for species Kokkos::fence(); diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 886d9d00609..29aaca421ea 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -30,18 +30,15 @@ struct srfEmissFunctions { struct srfEmissData { srfEmissData() = default; - srfEmissData(const int ncol_, const int nsectors_) : - ncols(ncol_), - nsectors(nsectors_) - { + srfEmissData(const int ncol_, const int nsectors_) + : ncols(ncol_), nsectors(nsectors_) { init(ncols, nsectors, true); } void init(const int ncol, const int nsector, const bool allocate) { - ncols = ncol; + ncols = ncol; nsectors = nsector; - if(allocate) - emiss_sectors = view_2d("AllSectors", nsectors, ncols); + if(allocate) emiss_sectors = view_2d("AllSectors", nsectors, ncols); } // srfEmissData init // Basic spatial dimensions of the data diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 7a677bbdca3..dfe1190cdb9 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,7 +57,8 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(std::vector::size_type icomp = 0; icomp < sector_names.size(); ++icomp) { + for(std::vector::size_type icomp = 0; + icomp < sector_names.size(); ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); @@ -87,8 +88,9 @@ srfEmissFunctions::create_srfEmiss_data_reader( template template -ScalarX srfEmissFunctions::linear_interp( - const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { +ScalarX srfEmissFunctions::linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t) { return (1 - t) * x0 + t * x1; } // linear_interp @@ -212,7 +214,8 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = srfEmiss_horiz_interp.get_tgt_field(i).get_view(); - const auto emiss = Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); + const auto emiss = + Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); Kokkos::deep_copy(emiss, sector); } From 245571a6e73fba2f3539e49c8442a767a874f61b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:33:00 -0700 Subject: [PATCH 48/65] Store size in local in to avoid declaring long size_type datatype --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index dfe1190cdb9..7b5195c6943 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -57,8 +57,9 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; - for(std::vector::size_type icomp = 0; - icomp < sector_names.size(); ++icomp) { + const int sector_size = sector_names.size(); + for(int icomp = 0; + icomp < sector_size; ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From 029ba9ccf009550d017f67f69c9e931393d3ef4e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:34:42 -0700 Subject: [PATCH 49/65] clang format --- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 7b5195c6943..2d29075e41a 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -58,8 +58,7 @@ srfEmissFunctions::create_horiz_remapper( std::vector field_emiss_sectors; const int sector_size = sector_names.size(); - for(int icomp = 0; - icomp < sector_size; ++icomp) { + for(int icomp = 0; icomp < sector_size; ++icomp) { auto comp_name = sector_names[icomp]; // set and allocate fields Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); From a816871c21f9550799d751cd69e6ed1038dec076 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 15:40:54 -0700 Subject: [PATCH 50/65] Revert "Merge branch 'master' into singhbalwinder/branch_mjs/mam4xx/emissionsIntegration" This reverts commit c2f7d298d93a06ad3c75a4d42f5139d08c7ed528, reversing changes made to 2d481948a48c0d9e006a0c2038328ca3642c499d. --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- cime_config/machines/config_machines.xml | 51 ++-- .../cmake/machine-files/anlgce-ub22.cmake | 12 - .../eamxx/cmake/machine-files/anlgce.cmake | 6 +- components/eamxx/scripts/machines_specs.py | 4 - .../src/dynamics/homme/eamxx_homme_iop.cpp | 259 +++++++++--------- ...mxx_mam_microphysics_process_interface.cpp | 4 +- components/eamxx/src/physics/p3/p3_iso_c.f90 | 19 +- .../atm_process/atmosphere_process_hash.cpp | 4 +- .../eamxx/src/share/io/scorpio_output.cpp | 35 +-- .../iop/intensive_observation_period.cpp | 12 +- 13 files changed, 181 insertions(+), 231 deletions(-) delete mode 100644 components/eamxx/cmake/machine-files/anlgce-ub22.cmake diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 9ea25ae1864..ccca0c479f2 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index e6fb53ba29f..abd8b92e4f6 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index e0a8e19f9c0..5ecdf6dec00 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.1.0 with: python-version: "3.11" - name: Run unit tests diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index eeb80756bb8..6765d000e26 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -368,7 +368,6 @@ nvidia aocc cudatoolkit - cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -379,24 +378,24 @@ - PrgEnv-gnu/8.5.0 - gcc-native/12.3 + PrgEnv-gnu/8.3.3 + gcc/11.2.0 PrgEnv-nvidia - nvidia/24.5 + nvidia/22.7 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - gcc-native-mixed/12.3 + gcc-mixed/11.2.0 @@ -408,12 +407,12 @@ - cray-libsci/23.12.5 - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 + cray-libsci/23.02.1.1 + craype/2.7.20 + cray-mpich/8.1.25 + cray-hdf5-parallel/1.12.2.3 + cray-netcdf-hdf5parallel/4.9.0.3 + cray-parallel-netcdf/1.12.3.3 cmake/3.24.3 @@ -678,7 +677,6 @@ nvidia aocc cudatoolkit - cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -689,24 +687,26 @@ - PrgEnv-gnu/8.5.0 - gcc-native/12.3 + PrgEnv-gnu/8.3.3 + gcc/11.2.0 + PrgEnv-nvidia - nvidia/24.5 + nvidia/23.9 - cudatoolkit/12.2 + cudatoolkit/11.7 + craype-accel-nvidia80 - cudatoolkit/12.2 + cudatoolkit/11.7 craype-accel-nvidia80 - gcc-native-mixed/12.3 @@ -718,13 +718,20 @@ - cray-libsci/23.12.5 + cray-libsci/23.02.1.1 + craype/2.7.20 + cray-mpich/8.1.25 + cray-hdf5-parallel/1.12.2.3 + cray-netcdf-hdf5parallel/4.9.0.3 + cray-parallel-netcdf/1.12.3.3 + cmake/3.24.3 + evp-patch diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake deleted file mode 100644 index e79f7001547..00000000000 --- a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake +++ /dev/null @@ -1,12 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -# Remove this if you are using a resource manager (slurm etc) -set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") - -# EKAT MPI settings -set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") -set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index e79f7001547..079000059d1 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) + # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 43e5ebfa083..9536d415c0d 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,10 +82,6 @@ ["mpicxx","mpifort","mpicc"], "", ""), - "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], - ["mpicxx","mpifort","mpicc"], - "", - ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index 74579df552b..eaf65a69cda 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,14 +10,14 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" +#include "ElementOps.hpp" +#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" +#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" -// SCREAM includes -#include "share/util/scream_common_physics_functions.hpp" - // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -222,7 +222,11 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - using PF = PhysicsFunctions; + + using EOS = Homme::EquationOfState; + using ElementOps = Homme::ElementOps; + using KV = Homme::KernelVariables; + using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -259,7 +263,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states + // Homme element states and EOS/EO classes auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -268,6 +272,11 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); + EOS eos; + eos.init(params.theta_hydrostatic_mode, hvcoord); + + ElementOps elem_ops; + elem_ops.init(hvcoord); const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); // Load data from IOP files, if necessary @@ -310,66 +319,92 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for eamxx - const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + // Team policy and workspace manager for both homme and scream + // related loops. We need separate policies since hommexx functions used here + // assume they are called inside nested loops for elements and Gaussian points, + // whereas EAMxx function we use expects a single level of parallelism + // for elements and Guassian points. + // TODO: scream::ColumnOps functions could take an arbitary loop boundary + // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that + // all 3 kernel launches here could be combined. + const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); + const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); + WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); + WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); view_Nd + rstar ("rstar", nelem, NGP, NGP, NLEV), + exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing temperature - auto compute_temperature = [&] () { - Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; + // Lambda for computing rstar, exner, and temperature from Hommexx + auto compute_homme_states = [&] () { + Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - // Compute temperature from virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - auto T_val = vtheta_dp_i(k); - T_val /= dp3d_i(k); - T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); - temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); + auto ws = homme_wsm.get_workspace(team); + auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); + uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto pnh_i = ekat::subview(pnh, idx); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); + Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); + Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); + Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); + + // Compute exner from EOS + if (theta_hydrostatic_mode) { + auto hydro_p_int = ws.take("hydro_p_int"); + Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); + elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); + eos.compute_exner(kv, pnh_scalar, exner_scalar); + ws.release(hydro_p_int); + } else { + eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); + } + + // Get the temperature from dynamics states + elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); }); // Release WS views - ws.release_many_contiguous<1>({&pmid}); + ws.release_macro_block(pnh_slot, NGP*NGP); }); }; - // Preprocess some homme states to get temperature - compute_temperature(); + // Preprocess some homme states to get temperature and exner + compute_homme_states(); Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); + auto ws = eamxx_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -414,57 +449,44 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid, pint, pdel; - ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, - {&pmid, &pint, &pdel}); + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - auto s_pint = ekat::scalarize(pint); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { - s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; - if (k < total_levels) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - } - }); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - team.team_barrier(); + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Convert updated temperature back to psuedo density virtual potential temperature - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + // Recompute rstar with updated qv, and convert updated temperature back to potential temperature + elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + }); }); - - // Release WS views - ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); if (iop_nudge_tq or iop_nudge_uv) { @@ -472,8 +494,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute temperature - compute_temperature(); + // Compute rstar, exner and temperature from Hommexx + compute_homme_states(); Kokkos::fence(); } @@ -549,36 +571,25 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_iop, + policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - - const int ie = team.league_rank()/(NGP*NGP); - const int igp = (team.league_rank()/NGP)%NGP; - const int jgp = team.league_rank()%NGP; - - // Get temp views from workspace - auto ws = iop_wsm.get_workspace(team); - uview_1d pmid; - ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); - - auto ps_i = ps_dyn(ie, igp, jgp); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - // Compute reference pressures and layer thickness. - // TODO: Allow geometry data to allocate packsize - auto s_pmid = ekat::scalarize(pmid); - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { - s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; - }); - team.team_barrier(); - - if (iop_nudge_tq or iop_nudge_uv) { - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { + KV kv(team); + const int ie = team.league_rank(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto rstar_i = ekat::subview(rstar, ie, igp, jgp); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high @@ -588,26 +599,22 @@ apply_iop_forcing(const Real dt) for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + and + pressure_from_iop >= iop_nudge_tq_high*100); } qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to virtual potential temperature - const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); - vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); + // Convert updated temperature back to potential temperature + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); } }); - } - - // Release WS views - ws.release_many_contiguous<1>({&pmid}); + }); }); } } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 8b93936148c..7ea2adfddb6 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,8 +317,6 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs - auto some_step = step_; - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index @@ -450,7 +448,7 @@ void MAMMicrophysics::run_impl(const double dt) { impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) - impl::modal_aero_amicphys_intr(config.amicphys, some_step, dt, t, pmid, pdel, + impl::modal_aero_amicphys_intr(config.amicphys, step_, dt, t, pmid, pdel, zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, vmr_precldchem, vmrcw_precldchem, vmr_tendbb, vmrcw_tendbb, dgncur_a, dgncur_awet, diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index ea0a18411c1..d75af1cb2ed 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=512), intent(out) :: string + character(kind=c_char, len=256), intent(inout) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,17 +69,10 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. -#ifdef SCREAM_DOUBLE_PRECISION - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR -#else - mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR - revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR - vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR - vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR -#endif + call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") + call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") + call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") + call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") if (write_tables) then call p3_init_b() diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 37cb251d779..9fb5726084f 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index a105e244d76..4cb3f50403a 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,42 +1038,11 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { - std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - if (mode==scorpio::FileMode::Append) { - // Similar to the regular fields above, check that the var is in the file, and has the right properties - EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), - "Error! Cannot append, due to variable missing from the file.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n"); - const auto& var = scorpio::get_var(filename,name); - EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, - "Error! Cannot append, due to variable dimensions mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var dims : " + ekat::join(vec_of_dims,",") + "\n" - " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); - EKAT_REQUIRE_MSG (var.units==unitless, - "Error! Cannot append, due to variable units mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var units: " + unitless + "\n" - " - var units from file: " + var.units + "\n"); - EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, - "Error! Cannot append, due to time dependency mismatch.\n" - " - filename : " + filename + "\n" - " - varname : " + name + "\n" - " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" - " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); - } else { - // Note, unlike with regular output variables, for the average counting - // variables we don't need to add all of the extra metadata. So we simply - // define the variable. - scorpio::define_var(filename, name, unitless, vec_of_dims, - "real",fp_precision, m_add_time_dim); - } + scorpio::define_var(filename, name, "unitless", vec_of_dims, + "real",fp_precision, m_add_time_dim); } } } // register_variables diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 32b120c5700..76e039e65ab 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -270,9 +270,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),(Real)0.1); + m_params.get("target_latitude"); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - std::max(m_params.get("target_longitude"),(Real)0.1); + m_params.get("target_longitude"); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -543,14 +543,6 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); - // Read in IOP lev data - auto data = iop_file_pressure.get_view().data(); - scorpio::read_var(iop_file,"lev",data); - - // Convert to pressure to millibar (file gives pressure in Pa) - for (int ilev=0; ilev(); From a7e719109b41762b6c63eb572a06b10f08351f4c Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 16:40:41 -0700 Subject: [PATCH 51/65] Rebases and update mam4 submodule to point to its latest master --- externals/mam4xx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/mam4xx b/externals/mam4xx index 5495c27a6df..2eee1988b1e 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 5495c27a6df7ae4e958d67077abc26d4c7e5765d +Subproject commit 2eee1988b1e6488c904e0a28564bdcadfa190d34 From f1b2ab67274e7120808df834bc82ae7e38ebc9b3 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Tue, 13 Aug 2024 21:10:12 -0700 Subject: [PATCH 52/65] Adds a multi-process test with srf emiss and constituent fluxes --- .../mam4xx/srf_online_emiss/shell_commands | 13 ----- .../multi-process/physics_only/CMakeLists.txt | 1 + .../CMakeLists.txt | 53 +++++++++++++++++++ .../input.yaml | 45 ++++++++++++++++ .../output.yaml | 41 ++++++++++++++ 5 files changed, 140 insertions(+), 13 deletions(-) delete mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml create mode 100644 components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands deleted file mode 100644 index 6995cd8b856..00000000000 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss/shell_commands +++ /dev/null @@ -1,13 +0,0 @@ - -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" - -#modify initial condition file to get aerosol species ICs -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b - -# Add spa as RRTMG needs spa -$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b - - - diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index d6ed13dfb0a..bb23911ff1c 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -11,6 +11,7 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) add_subdirectory(mam/p3_mam4_wetscav) add_subdirectory(mam/shoc_cldfrac_p3_wetscav) + add_subdirectory(mam/mam4_srf_online_emiss_mam4_constituent_fluxes) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt new file mode 100644 index 00000000000..8d2f2653387 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt @@ -0,0 +1,53 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME mam4_srf_online_emiss_mam4_constituent_fluxes) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS mam + LABELS physics mam4_srf_online_emiss mam4_constituent_fluxes + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS physics mam4_srf_online_emiss mam4_constituent_fluxes + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml new file mode 100644 index 00000000000..45fac36611e --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -0,0 +1,45 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_srf_online_emiss, mam4_constituent_fluxes] + schedule_type: Sequential + mam4_srf_online_emiss: + # MAM4xx-Surface-Emissions + srf_remap_file: "" + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml new file mode 100644 index 00000000000..15e60e91517 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml @@ -0,0 +1,41 @@ +%YAML 1.1 +--- +filename_prefix: shoc_mam4_drydep_output +Averaging Type: Instant +Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - constituent_fluxes + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... From 2bf266178d60a74c08ad005632e8b14c2de7f81b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 14 Aug 2024 14:06:41 -0700 Subject: [PATCH 53/65] Adds testmods and files for ne4pg2 CIME simulation --- .../cime_config/namelist_defaults_scream.xml | 29 ++++++++++--------- .../shell_commands | 17 +++++++++++ ...and_online_emissions_process_interface.cpp | 2 +- .../input.yaml | 2 ++ 4 files changed, 35 insertions(+), 15 deletions(-) create mode 100644 components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index e8e062cfaeb..12095edff39 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -242,8 +242,7 @@ be lost if SCREAM_HACK_XML is not enabled. - - + @@ -266,22 +265,24 @@ be lost if SCREAM_HACK_XML is not enabled. - - + + - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DMSflux.2010.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20190220.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_bc_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a2_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_num_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_pom_a4_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a1_surf_1x1_2010_clim_c20190821.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/DECK_ne30/cmip6_mam4_so4_a2_surf_1x1_2010_clim_c20190821.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240814.nc + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands new file mode 100644 index 00000000000..05b0b495411 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -0,0 +1,17 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +#Update IC file and add the processes +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b + + + diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index a4ac4778982..f2a68ee6b5a 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -40,7 +40,7 @@ void MAMSrfOnlineEmiss::set_grids( grid_name); // Surface emissions remapping file - std::string srf_map_file = m_params.get("srf_remap_file"); + auto srf_map_file = m_params.get("srf_remap_file", ""); //-------------------------------------------------------------------- // Init dms srf emiss data structures diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 45fac36611e..24c1d9f886e 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -39,6 +39,8 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 0.0 + # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] From 5879c571b60dc1ed89a78bdd3b2b4fb6f55daa97 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 14 Aug 2024 23:21:05 -0700 Subject: [PATCH 54/65] Revertes codes that were changed after the rebase --- .github/workflows/e3sm-gh-pages.yml | 2 +- .github/workflows/eamxx-gh-pages.yml | 2 +- .github/workflows/eamxx_default_files.yml | 2 +- cime_config/machines/config_machines.xml | 51 ++-- .../cmake/machine-files/anlgce-ub22.cmake | 12 + .../eamxx/cmake/machine-files/anlgce.cmake | 6 +- components/eamxx/scripts/machines_specs.py | 4 + .../src/dynamics/homme/eamxx_homme_iop.cpp | 259 +++++++++--------- ...mxx_mam_microphysics_process_interface.cpp | 2 + components/eamxx/src/physics/p3/p3_iso_c.f90 | 19 +- .../atm_process/atmosphere_process_hash.cpp | 4 +- .../eamxx/src/share/io/scorpio_output.cpp | 35 ++- .../src/share/io/scream_scorpio_interface.cpp | 6 +- .../iop/intensive_observation_period.cpp | 12 +- .../single-process/mam/optics/CMakeLists.txt | 1 - 15 files changed, 233 insertions(+), 184 deletions(-) create mode 100644 components/eamxx/cmake/machine-files/anlgce-ub22.cmake diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ccca0c479f2..9ea25ae1864 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -31,7 +31,7 @@ jobs: - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}." - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" - name: Install python deps diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index abd8b92e4f6..e6fb53ba29f 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -54,7 +54,7 @@ jobs: echo "= The job was automatically triggered by a ${{github.event_name}} event." - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 5ecdf6dec00..e0a8e19f9c0 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -21,7 +21,7 @@ jobs: show-progress: false submodules: false - name: Set up Python 3.11 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.11" - name: Run unit tests diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 6765d000e26..eeb80756bb8 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -368,6 +368,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -378,24 +379,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/22.7 + nvidia/24.5 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - gcc-mixed/11.2.0 + gcc-native-mixed/12.3 @@ -407,12 +408,12 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + cray-libsci/23.12.5 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 @@ -677,6 +678,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -687,26 +689,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 - + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 - cudatoolkit/11.7 - + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 + gcc-native-mixed/12.3 @@ -718,20 +718,13 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 - evp-patch diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake new file mode 100644 index 00000000000..e79f7001547 --- /dev/null +++ b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake @@ -0,0 +1,12 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Remove this if you are using a resource manager (slurm etc) +set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") + +# EKAT MPI settings +set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index 079000059d1..e79f7001547 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) - # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 9536d415c0d..43e5ebfa083 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -82,6 +82,10 @@ ["mpicxx","mpifort","mpicc"], "", ""), + "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index eaf65a69cda..74579df552b 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,14 +10,14 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" -#include "ElementOps.hpp" -#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" -#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" +// SCREAM includes +#include "share/util/scream_common_physics_functions.hpp" + // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -222,11 +222,7 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - - using EOS = Homme::EquationOfState; - using ElementOps = Homme::ElementOps; - using KV = Homme::KernelVariables; - + using PF = PhysicsFunctions; using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -263,7 +259,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states and EOS/EO classes + // Homme element states auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -272,11 +268,6 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - EOS eos; - eos.init(params.theta_hydrostatic_mode, hvcoord); - - ElementOps elem_ops; - elem_ops.init(hvcoord); const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); // Load data from IOP files, if necessary @@ -319,92 +310,66 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for both homme and scream - // related loops. We need separate policies since hommexx functions used here - // assume they are called inside nested loops for elements and Gaussian points, - // whereas EAMxx function we use expects a single level of parallelism - // for elements and Guassian points. - // TODO: scream::ColumnOps functions could take an arbitary loop boundary - // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that - // all 3 kernel launches here could be combined. - const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); - const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + // Team policy and workspace manager for eamxx + const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); - WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); + WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); view_Nd - rstar ("rstar", nelem, NGP, NGP, NLEV), - exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing rstar, exner, and temperature from Hommexx - auto compute_homme_states = [&] () { - Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + // Lambda for computing temperature + auto compute_temperature = [&] () { + Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = homme_wsm.get_workspace(team); - auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); - uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto pnh_i = ekat::subview(pnh, idx); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); - Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); - Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - - // Compute exner from EOS - if (theta_hydrostatic_mode) { - auto hydro_p_int = ws.take("hydro_p_int"); - Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); - elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); - eos.compute_exner(kv, pnh_scalar, exner_scalar); - ws.release(hydro_p_int); - } else { - eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); - } - - // Get the temperature from dynamics states - elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + // Compute temperature from virtual potential temperature + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { + auto T_val = vtheta_dp_i(k); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); }); // Release WS views - ws.release_macro_block(pnh_slot, NGP*NGP); + ws.release_many_contiguous<1>({&pmid}); }); }; - // Preprocess some homme states to get temperature and exner - compute_homme_states(); + // Preprocess some homme states to get temperature + compute_temperature(); Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -449,44 +414,57 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; + // Get temp views from workspace + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + + team.team_barrier(); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Recompute rstar with updated qv, and convert updated temperature back to potential temperature - elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); - }); + // Convert updated temperature back to psuedo density virtual potential temperature + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); + + // Release WS views + ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); if (iop_nudge_tq or iop_nudge_uv) { @@ -494,8 +472,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute rstar, exner and temperature from Hommexx - compute_homme_states(); + // Compute temperature + compute_temperature(); Kokkos::fence(); } @@ -571,25 +549,36 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_homme, + policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { + + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; + + // Get temp views from workspace + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + if (iop_nudge_tq or iop_nudge_uv) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high @@ -599,22 +588,26 @@ apply_iop_forcing(const Real dt) for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + and + pressure_from_iop >= iop_nudge_tq_high*100); } qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to potential temperature - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); } }); - }); + } + + // Release WS views + ws.release_many_contiguous<1>({&pmid}); }); } } diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7ea2adfddb6..f8a34bd90d5 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,6 +317,8 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index d75af1cb2ed..ea0a18411c1 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=256), intent(inout) :: string + character(kind=c_char, len=512), intent(out) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,10 +69,17 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. - call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") - call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") - call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") - call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") +#ifdef SCREAM_DOUBLE_PRECISION + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR +#else + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR +#endif if (write_tables) then call p3_init_b() diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 9fb5726084f..37cb251d779 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,7 +129,7 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16llx (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lx (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), i, gaccum[i], label.c_str()); } @@ -140,7 +140,7 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16llx (%s)\n", + fprintf(stderr, "bfbhash> %14d %16lx (%s)\n", timestamp().get_num_steps(), gaccum, label.c_str()); } diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 4cb3f50403a..a105e244d76 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,11 +1038,42 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { + std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - scorpio::define_var(filename, name, "unitless", vec_of_dims, - "real",fp_precision, m_add_time_dim); + if (mode==scorpio::FileMode::Append) { + // Similar to the regular fields above, check that the var is in the file, and has the right properties + EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), + "Error! Cannot append, due to variable missing from the file.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n"); + const auto& var = scorpio::get_var(filename,name); + EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, + "Error! Cannot append, due to variable dimensions mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var dims : " + ekat::join(vec_of_dims,",") + "\n" + " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + EKAT_REQUIRE_MSG (var.units==unitless, + "Error! Cannot append, due to variable units mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var units: " + unitless + "\n" + " - var units from file: " + var.units + "\n"); + EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, + "Error! Cannot append, due to time dependency mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" + " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); + } else { + // Note, unlike with regular output variables, for the average counting + // variables we don't need to add all of the extra metadata. So we simply + // define the variable. + scorpio::define_var(filename, name, unitless, vec_of_dims, + "real",fp_precision, m_add_time_dim); + } } } } // register_variables diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 2518907ec52..8d2f64994dd 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -361,9 +361,9 @@ void finalize_subsystem () "Error! PIO subsystem was already finalized.\n"); for (auto& it : s.files) { - // EKAT_REQUIRE_MSG (it.second.num_customers==0, - // "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" - // " - filename: " + it.first + "\n"); + EKAT_REQUIRE_MSG (it.second.num_customers==0, + "Error! ScorpioSession::finalize called, but a file is still in use elsewhere.\n" + " - filename: " + it.first + "\n"); } s.files.clear(); diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index 76e039e65ab..32b120c5700 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -270,9 +270,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - m_params.get("target_latitude"); + std::max(m_params.get("target_latitude"),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - m_params.get("target_longitude"); + std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -543,6 +543,14 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); + // Read in IOP lev data + auto data = iop_file_pressure.get_view().data(); + scorpio::read_var(iop_file,"lev",data); + + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilev(); diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt index 50f2cfa68a3..0a3a0b877c5 100644 --- a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -40,7 +40,6 @@ set (TEST_INPUT_FILES scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc scream/mam4xx/physprops/poly_rrtmg_c20240206.nc - # scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) From 1d796b40a8362b503206274cfba423dd5c4577ee Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 15 Aug 2024 10:00:57 -0700 Subject: [PATCH 55/65] Changes emission files that use conservative mapping --- .../cime_config/namelist_defaults_scream.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 12095edff39..87791d0b7d0 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -271,14 +271,14 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc From 731618181e6c91c92335eb1b535f706ed654657a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 11:05:22 -0700 Subject: [PATCH 56/65] Fixes multi process test a typo in aci interface --- .../src/physics/mam/eamxx_mam_aci_process_interface.cpp | 8 ++++---- .../output.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index 7800a43c252..ef2c0f2b288 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -253,7 +253,7 @@ void MAMAci::set_grids( frz_unit, grid_name); // heterogeneous freezing by deposition nucleation [cm^-3 s^-1] - add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, + add_field("hetfrz_deposition_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); } // function set_grids ends @@ -390,8 +390,8 @@ void MAMAci::initialize_impl(const RunType run_type) { get_field_out("hetfrz_immersion_nucleation_tend").get_view(); hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = - get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + hetfrz_deposition_nucleation_tend_ = + get_field_out("hetfrz_deposition_nucleation_tend").get_view(); //--------------------------------------------------------------------------------- // Allocate memory for the class members @@ -642,7 +642,7 @@ void MAMAci::run_impl(const double dt) { team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, // ## output to be used by the other processes ## hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, - hetfrz_depostion_nucleation_tend_, + hetfrz_deposition_nucleation_tend_, // work arrays diagnostic_scratch_); diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml index 15e60e91517..c9cad5e2761 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: shoc_mam4_drydep_output +filename_prefix: mam4_srf_online_emiss_mam4_constituent_fluxes_output Averaging Type: Instant Field Names: - bc_a1 From c0a608e419600b9260f62f75030c03a9149d459e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 15:32:22 -0700 Subject: [PATCH 57/65] Adds ne30pg2 emissions files and mapping files for fine resolutions --- .../cime_config/namelist_defaults_scream.xml | 18 ++++++++++++++++++ .../mam/eamxx_mam_aci_process_interface.hpp | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 87791d0b7d0..a785fb32291 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -279,6 +279,24 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + + + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120np4_mono_20220502.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index fc930ea11d7..63f87dd9f24 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -123,7 +123,7 @@ class MAMAci final : public scream::AtmosphereProcess { // added correctly to the cloud-micorphysics scheme. view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; - view_2d hetfrz_depostion_nucleation_tend_; + view_2d hetfrz_deposition_nucleation_tend_; view_2d diagnostic_scratch_[hetro_scratch_]; From 4fa577ebced271a90ee7c29cb308eada86426d6f Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 16 Aug 2024 19:12:45 -0700 Subject: [PATCH 58/65] Removes debug print statements --- ...eamxx_mam_constituent_fluxes_interface.cpp | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index c74dbd7e4fa..c3a33687dad 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -1,4 +1,3 @@ -#include #include #include namespace scream { @@ -282,30 +281,10 @@ void MAMConstituentFluxes::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - auto start = std::chrono::steady_clock::now(); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, constituent_fluxes_, // output wet_aero_); - auto stop = std::chrono::steady_clock::now(); - auto duration = (stop - start); - - // To get the value of duration use the count() - // member function on the duration object - printf("TIME:%e\n", - std::chrono::duration(duration).count()); - - for(int icnst = 0; icnst < 6; ++icnst) { - auto host_view = Kokkos::create_mirror_view(wet_aero_.gas_mmr[icnst]); - Kokkos::deep_copy(host_view, wet_aero_.gas_mmr[icnst]); - printf("BEFORE:::%e, %i\n", host_view(0, 71), icnst + 9); - } - } // run_impl ends // ============================================================================= From 87ea43c4f5cefdf12bd1ee869c3508230db1f8b1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 17 Aug 2024 15:20:01 -0700 Subject: [PATCH 59/65] Edits and adds some comments to provide clarity and TODOs --- .../mam/eamxx_mam_constituent_fluxes_functions.hpp | 3 +++ .../mam/eamxx_mam_constituent_fluxes_interface.cpp | 8 ++++---- .../mam/eamxx_mam_constituent_fluxes_interface.hpp | 2 +- ...mxx_mam_srf_and_online_emissions_process_interface.cpp | 4 +++- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 2 ++ 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index e7ffb6376c3..12c9c172f17 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -33,6 +33,9 @@ void update_gas_aerosols_using_constituents( // Create a policy to loop over columns annd number of constituents // to update + // FIXME: TODO:We don't need a team for "nconstituents", so we can make the + // kookos_for + // simple by using just ncols const auto policy = ekat::ExeSpaceUtils:: get_default_team_policy(ncol, nconstituents); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index c3a33687dad..96d26ca22f9 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -267,13 +267,13 @@ void MAMConstituentFluxes::run_impl(const double dt) { // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in // E3SM. In EAMxx, all constituents are considered "wet" (or have wet - // mixing ratios), so we are *not* doing any wet to dry conversions in the - // "preprocess" . We are simply updating the MAM4xx tracers using the - // "constituent fluxes". + // mixing ratios), we are *not* doing any wet to dry conversions in the + // "preprocess" for this process. We are simply updating the MAM4xx + // tracers using the "constituent fluxes". // We are converting wet atm to dry atm. Since we do not use or update // any of the water constituents (qc, qv, qi etc.), we should be okay // to do this conversion. We need to do this conversion as our function - // are build following HAERO data structures. + // are built following HAERO data structures. // ------------------------------------------------------------------- // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index 7e77fccf2af..c68065f2178 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -27,7 +27,7 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { mam_coupling::DryAtmosphere dry_atm_; // aerosol state variables - mam_coupling::AerosolState wet_aero_; //, dry_aero_; + mam_coupling::AerosolState wet_aero_; // buffer for sotring temporary variables mam_coupling::Buffer buffer_; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index f2a68ee6b5a..850a82d0896 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -42,6 +42,8 @@ void MAMSrfOnlineEmiss::set_grids( // Surface emissions remapping file auto srf_map_file = m_params.get("srf_remap_file", ""); + // FIXME: We can extract the following info about each species + // in a separate hpp file //-------------------------------------------------------------------- // Init dms srf emiss data structures //-------------------------------------------------------------------- @@ -221,7 +223,7 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - // Zero output + // Zero-out output Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); // Gather time and state information for interpolation diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 2d29075e41a..b8ac7de768d 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -28,6 +28,8 @@ srfEmissFunctions::create_horiz_remapper( const int ncols_model = model_grid->get_num_global_dofs(); std::shared_ptr remapper; + // if the file's grid is same as model's native grid, we identity remapper + // (i.e., no interpolation) if(ncols_data == ncols_model) { remapper = std::make_shared( horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); From 050d14990f5420c51fa55a03be332d49854d4a7e Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 28 Aug 2024 16:12:23 -0700 Subject: [PATCH 60/65] Fixes namelist to enable fine grid simulations --- .../cime_config/namelist_defaults_scream.xml | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index a785fb32291..dadb6dbba35 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -269,7 +269,7 @@ be lost if SCREAM_HACK_XML is not enabled. - + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc @@ -279,20 +279,19 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc - - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120np4_mono_20220502.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc From d8d89e69a472ffb5e8f005bfdd20908e97c7a636 Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Thu, 29 Aug 2024 11:48:28 -0700 Subject: [PATCH 61/65] Update shell_commands to remove spa --- .../mam4xx/srf_online_emiss_constituent_fluxes/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands index 05b0b495411..f47a1729ed7 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -11,7 +11,7 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/u #Update IC file and add the processes #------------------------------------------------------ $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b -$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,spa,rrtmgp,mam4_srf_online_emiss" -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,rrtmgp,mam4_srf_online_emiss" -b From 4b573528ff76ff23b3613792db02ee0dea9a4dbc Mon Sep 17 00:00:00 2001 From: singhbalwinder Date: Thu, 29 Aug 2024 12:50:13 -0700 Subject: [PATCH 62/65] Adds units for mass and number ofr constituent fluxes --- .../src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 96d26ca22f9..3a1a1d39b63 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -86,6 +86,7 @@ void MAMConstituentFluxes::set_grids( add_field("phis", scalar2d, m2 / s2, grid_name); // Constituent fluxes at the surface (gasses and aerosols) + //[units: kg/m2/s (mass) or #/m2/s (number)] add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, grid_name); From 215d225dcfc04c4e9ebffa9b63c05d77b993e03a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 31 Aug 2024 23:40:20 -0700 Subject: [PATCH 63/65] Fixes path for files and change c-style strings to std::strings --- ...eamxx_mam_constituent_fluxes_interface.cpp | 30 ++++++++++--------- .../CMakeLists.txt | 18 +++++------ .../input.yaml | 18 +++++------ .../mam/emissions/CMakeLists.txt | 18 +++++------ .../single-process/mam/emissions/input.yaml | 18 +++++------ 5 files changed, 52 insertions(+), 50 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 3a1a1d39b63..543a445da98 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -100,7 +100,7 @@ void MAMConstituentFluxes::set_grids( // number (n) mixing ratios for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = + const std::string int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(mode); add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, "tracers"); @@ -108,31 +108,31 @@ void MAMConstituentFluxes::set_grids( // cloudborne aerosol tracers of interest: number (n) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - const char *cld_nmr_field_name = + const std::string cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(mode); add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const std::string int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(mode, a); - if(strlen(int_mmr_field_name) > 0) { + if(not int_mmr_field_name.empty()) { add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are // NOT advected - const char *cld_mmr_field_name = + const std::string cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(mode, a); - if(strlen(cld_mmr_field_name) > 0) { + if(not cld_mmr_field_name.empty()) { add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); } } // end for loop num species } // end for loop for num modes for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, "tracers"); } // end for loop num gases @@ -217,36 +217,38 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { // number (n) mixing ratios for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { // interstitial aerosol tracers of interest: number (n) mixing ratios - const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + const std::string int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(m); wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); // cloudborne aerosol tracers of interest: number (n) mixing ratios - const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + const std::string cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(m); wet_aero_.cld_aero_nmr[m] = get_field_out(cld_nmr_field_name).get_view(); for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { // (interstitial) aerosol tracers of interest: mass (q) mixing ratios - const char *int_mmr_field_name = + const std::string int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); - if(strlen(int_mmr_field_name) > 0) { + if(not int_mmr_field_name.empty()) { wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); } // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios - const char *cld_mmr_field_name = + const std::string cld_mmr_field_name = mam_coupling::cld_aero_mmr_field_name(m, a); - if(strlen(cld_mmr_field_name) > 0) { + if(not cld_mmr_field_name.empty()) { wet_aero_.cld_aero_mmr[m][a] = get_field_out(cld_mmr_field_name).get_view(); } } } for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { - const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); wet_aero_.gas_mmr[g] = get_field_out(gas_mmr_field_name).get_view(); } diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt index 8d2f2653387..886936ca084 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt @@ -22,15 +22,15 @@ GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 24c1d9f886e..55c62d69aa2 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -14,15 +14,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index 3d64bc5b2b2..c8e690b929f 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -27,15 +27,15 @@ GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - scream/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index 06be9ec8a88..e7af45178aa 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -13,15 +13,15 @@ atmosphere_processes: mam4_srf_online_emiss: # MAM4xx-Surface-Emissions srf_remap_file: "" - srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc - srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc - srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc - srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc grids_manager: Type: Mesh Free From b1fbda11b93ed28d292291b36db2f6059fa5bd1b Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 31 Aug 2024 23:56:03 -0700 Subject: [PATCH 64/65] Replaces preprocess interface code --- ...eamxx_mam_constituent_fluxes_interface.cpp | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 543a445da98..3efd57864fa 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -281,8 +281,26 @@ void MAMConstituentFluxes::run_impl(const double dt) { // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ // etc. - Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - Kokkos::fence(); + // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + // Kokkos::fence(); + + auto lambda = + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int i = team.league_rank(); // column index + compute_dry_mixing_ratios(team, wet_atm_, // in + dry_atm_, // out + i); // in + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, // in + dry_atm_, // out + i); // in + compute_updraft_velocities(team, wet_atm_, // in + dry_atm_, // out + i); // in + }; + Kokkos::parallel_for("mam_cfi_compute_updraft", scan_policy, lambda); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, constituent_fluxes_, From c6d07118118c5241ba8a923d513412b449bc9fb0 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sun, 1 Sep 2024 00:15:47 -0700 Subject: [PATCH 65/65] Removes preprocess function from hpp and a calls from cpp interface files --- ...eamxx_mam_constituent_fluxes_functions.hpp | 10 ++--- ...eamxx_mam_constituent_fluxes_interface.cpp | 32 ++++++--------- ...eamxx_mam_constituent_fluxes_interface.hpp | 41 ------------------- 3 files changed, 16 insertions(+), 67 deletions(-) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp index 12c9c172f17..602fc9ebf89 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -19,10 +19,7 @@ void update_gas_aerosols_using_constituents( static constexpr int pcnst = mam4::aero_model::pcnst; // Declare local variables - auto wet_aero_loc = wet_aero; - auto dry_atm_loc = dry_atm; - const int surface_lev = nlev - 1; - auto constituent_fluxes_loc = constituent_fluxes; + const int surface_lev = nlev - 1; // get the start index for gas species in the state_q array int istart = mam4::utils::gasses_start_ind(); @@ -34,8 +31,7 @@ void update_gas_aerosols_using_constituents( // Create a policy to loop over columns annd number of constituents // to update // FIXME: TODO:We don't need a team for "nconstituents", so we can make the - // kookos_for - // simple by using just ncols + // kookos_for simple by using just ncols const auto policy = ekat::ExeSpaceUtils:: get_default_team_policy(ncol, nconstituents); @@ -58,7 +54,7 @@ void update_gas_aerosols_using_constituents( atmosphere_for_column(dry_atm, // output icol); // input - // Form state%q like array + // Form state%q like array at surface level Real state_q_at_surf_lev[pcnst] = {}; mam4::utils::extract_stateq_from_prognostics( progs_at_col, haero_atm, // input diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp index 3efd57864fa..c168c1ef0e3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -253,53 +253,47 @@ void MAMConstituentFluxes::initialize_impl(const RunType run_type) { get_field_out(gas_mmr_field_name).get_view(); } - //----------------------------------------------------------------- - // Setup preprocessing and post processing - //----------------------------------------------------------------- - preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); - } // end initialize_impl() // ================================================================ // RUN_IMPL // ================================================================ void MAMConstituentFluxes::run_impl(const double dt) { - const auto scan_policy = ekat::ExeSpaceUtils< - KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); - // ------------------------------------------------------------------- // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in // E3SM. In EAMxx, all constituents are considered "wet" (or have wet // mixing ratios), we are *not* doing any wet to dry conversions in the - // "preprocess" for this process. We are simply updating the MAM4xx - // tracers using the "constituent fluxes". + // for this process. We are simply updating the MAM4xx tracers using the + // "constituent fluxes". // We are converting wet atm to dry atm. Since we do not use or update // any of the water constituents (qc, qv, qi etc.), we should be okay // to do this conversion. We need to do this conversion as our function // are built following HAERO data structures. // ------------------------------------------------------------------- - // preprocess input -- needs a scan for the calculation of dry_atm_, wet_aero_ - // etc. - // Kokkos::parallel_for("preprocess", scan_policy, preprocess_); - // Kokkos::fence(); - + // Compute vertical layer heights and updraft velocity. We need these to fully + // populate dry_atm_, so that we can form a HAERO atmosphere object. HAERO + // atmosphere object is used to for state%q like array. auto lambda = KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { - const int i = team.league_rank(); // column index + const int icol = team.league_rank(); // column index compute_dry_mixing_ratios(team, wet_atm_, // in dry_atm_, // out - i); // in + icol); // in team.team_barrier(); // vertical heights has to be computed after computing dry mixing ratios // for atmosphere compute_vertical_layer_heights(team, // in dry_atm_, // out - i); // in + icol); // in compute_updraft_velocities(team, wet_atm_, // in dry_atm_, // out - i); // in + icol); // in }; + // policy + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + Kokkos::parallel_for("mam_cfi_compute_updraft", scan_policy, lambda); update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp index c68065f2178..7d54761a297 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -69,47 +69,6 @@ class MAMConstituentFluxes final : public scream::AtmosphereProcess { // Finalize void finalize_impl(){/*Do nothing*/}; - // Atmosphere processes often have a pre-processing step that constructs - // required variables from the set of fields stored in the field manager. - // This functor implements this step, which is called during run_impl. - struct Preprocess { - Preprocess() = default; - // on host: initializes preprocess functor with necessary state data - void initialize(const int ncol, const int nlev, - const mam_coupling::WetAtmosphere &wet_atm, - const mam_coupling::DryAtmosphere &dry_atm) { - ncol_pre_ = ncol; - nlev_pre_ = nlev; - wet_atm_pre_ = wet_atm; - dry_atm_pre_ = dry_atm; - } - - KOKKOS_INLINE_FUNCTION - void operator()( - const Kokkos::TeamPolicy::member_type &team) const { - const int i = team.league_rank(); // column index - - compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); - team.team_barrier(); - // vertical heights has to be computed after computing dry mixing ratios - // for atmosphere - compute_vertical_layer_heights(team, dry_atm_pre_, i); - compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); - } // operator() - - // local variables for preprocess struct - // number of horizontal columns and vertical levels - int ncol_pre_, nlev_pre_; - - // local atmospheric and aerosol state data - mam_coupling::WetAtmosphere wet_atm_pre_; - mam_coupling::DryAtmosphere dry_atm_pre_; - }; // Preprocess - - private: - // preprocessing scratch pads - Preprocess preprocess_; - }; // MAMConstituentFluxes } // namespace scream