From 2de2c8dbfab3cd18b8d02f72ac1cc1df02156374 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 9 Sep 2024 16:31:05 -0600 Subject: [PATCH 01/19] RHS and Jacobian test for wall time measurements. --- src/examples/CMakeLists.txt | 3 +- src/examples/TChem_AerosolChemistry_RHSs.cpp | 386 +++++++++++++++++++ src/examples/TChem_AtmosphericChemistry.cpp | 22 +- 3 files changed, 402 insertions(+), 9 deletions(-) create mode 100644 src/examples/TChem_AerosolChemistry_RHSs.cpp diff --git a/src/examples/CMakeLists.txt b/src/examples/CMakeLists.txt index 4e3f811a..0e3580d1 100644 --- a/src/examples/CMakeLists.txt +++ b/src/examples/CMakeLists.txt @@ -13,6 +13,7 @@ LIST(APPEND TCHEM_ATM_EXAMPLE_SOURCES TChem_AtmosphericChemistryE3SM_CVODE.cpp TChem_AerosolChemistry_CVODE.cpp TChem_AerosolChemistry.cpp + TChem_AerosolChemistry_RHSs.cpp ) IF(TCHEM_ATM_ENABLE_TPL_KOKKOSKERNELS) LIST(APPEND TCHEM_ATM_EXAMPLE_SOURCES @@ -72,4 +73,4 @@ TARGET_LINK_LIBRARIES(${TCHEM_ATM_EXAMPLE_EXE} ${TCHEM_ATM_LINK_LIBRARIES}) INSTALL(TARGETS ${TCHEM_ATM_EXAMPLE_EXE} PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE DESTINATION "${CMAKE_INSTALL_PREFIX}/${TCHEM_ATM_INSTALL_EXAMPLE_PATH}") -ENDIF() \ No newline at end of file +ENDIF() diff --git a/src/examples/TChem_AerosolChemistry_RHSs.cpp b/src/examples/TChem_AerosolChemistry_RHSs.cpp new file mode 100644 index 00000000..2c013d6f --- /dev/null +++ b/src/examples/TChem_AerosolChemistry_RHSs.cpp @@ -0,0 +1,386 @@ +/* ===================================================================================== +TChem-atm version 1.0 +Copyright (2024) NTESS +https://github.com/sandialabs/TChem-atm + +Copyright 2024 National Technology & Engineering Solutions of Sandia, LLC +(NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +Government retains certain rights in this software. + +This file is part of TChem-atm. TChem-atm is open source software: you can redistribute +it and/or modify it under the terms of BSD 2-Clause License +(https://opensource.org/licenses/BSD-2-Clause). A copy of the licese is also +provided under the main directory + +Questions? Contact Oscar Diaz-Ibarra at , or + Mike Schmidt at , or + Cosmin Safta at + +Sandia National Laboratories, New Mexico/Livermore, NM/CA, USA +===================================================================================== +*/ +#include "TChem.hpp" +#include "TChem_CommandLineParser.hpp" +#include "TChem_Impl_AerosolChemistry.hpp" + +using ordinal_type = TChem::ordinal_type; +using real_type = TChem::real_type; + +int main(int argc, char *argv[]) { + std::string chemFile( "chem.yaml"); + std::string aeroFile("aero.yaml"); + std::string outputFile("AerosolChemistry.dat"); + std::string inputFile("None"); + std::string input_file_particles("None"); + std::string outputFileTimes("wall_times.json"); + int nBatch(1), team_size(-1), vector_size(-1); + bool verbose(true); + bool use_cloned_samples(false); + + /// parse command line arguments + TChem::CommandLineParser opts("This example computes the net production rates with a given state vector"); + opts.set_option("chemfile", "Chem file name e.g., chem.inp", + &chemFile); + opts.set_option("aerofile", "Aerosol Chem file name e.g., aero.yaml", + &aeroFile); + opts.set_option("inputfile", "Chem file name e.g., chem.inp", + &inputFile); + opts.set_option("inputfile_particles", "input file name e.g., chem.inp", + &input_file_particles); + opts.set_option("outputfile", "Output omega file name e.g., omega.dat", &outputFile); + opts.set_option("outputfile_times", "Wal times file name e.g., times.json", &outputFileTimes); + opts.set_option("team_thread_size", "time thread size ", &team_size); + opts.set_option("vector_thread_size", "vector thread size ", &vector_size); + opts.set_option("verbose", "If true, printout the first omega values", &verbose); + opts.set_option("batch_size", " number of batches or samples, e.g. 10 ", &nBatch); + opts.set_option( + "use_cloned_samples", "If true, one state vector will be cloned.", &use_cloned_samples); + + const bool r_parse = opts.parse(argc, argv); + if (r_parse) + return 0; // print help return + + Kokkos::initialize(argc, argv); + { + + // by default use input condition from chemFile + if (inputFile == "None") { + inputFile=chemFile; + } + // check first in chem file + if (input_file_particles == "None") { + input_file_particles=chemFile; + } + + const bool detail = false; + constexpr real_type zero =0; + + using host_device_type = typename Tines::UseThisDevice::type; + + using device_type = typename Tines::UseThisDevice::type; + using problem_type = TChem::Impl::AerosolChemistry_Problem; + + using real_type_0d_view_type = Tines::value_type_0d_view; + using real_type_1d_view_type = Tines::value_type_1d_view; + using real_type_2d_view_type = Tines::value_type_2d_view; + using real_type_3d_view_type = Tines::value_type_3d_view; + using real_type_2d_view_host_type = Tines::value_type_2d_view; + + + TChem::exec_space().print_configuration(std::cout, detail); + TChem::host_exec_space().print_configuration(std::cout, detail); + const auto exec_space_instance = TChem::exec_space(); + + /// construct kmd and use the view for testing + printf("kmd parsing ...\n"); + TChem::KineticModelData kmd(chemFile); + const auto kmcd = TChem::createNCAR_KineticModelConstData(kmd); + + printf("amd parsing ...\n"); + TChem::AerosolModelData amd(aeroFile, kmd); + const auto amcd = TChem::create_AerosolModelConstData(amd); + + const ordinal_type total_n_species =kmcd.nSpec + amcd.nSpec * amcd.nParticles; + const ordinal_type number_of_equations = problem_type::getNumberOfTimeODEs(kmcd, amcd); + const ordinal_type stateVecDim = + TChem::Impl::getStateVectorSize(total_n_species); + const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); + Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); + + printf("Number of Gas Species %d \n", kmcd.nSpec); + printf("Number of Gas Reactions %d \n", kmcd.nReac); + + printf("Number of Aerosol Species %d \n", amcd.nSpec); + printf("Number of Aerosol Particles %d \n", amcd.nParticles); + + printf("stateVecDim %d \n", stateVecDim); + + // read scenario condition from yaml file + real_type_2d_view_host_type state_scenario_host; + ordinal_type nbacth_files=1; + printf("conditions parsing ...\n"); + TChem::AtmChemistry::setScenarioConditions(inputFile, + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + printf("Number of const species %d \n", kmcd.nConstSpec); + + real_type_2d_view_host_type num_concentration_host; + real_type_2d_view_host_type state_host; + + // scenario particles + amd.scenarioConditionParticles(input_file_particles, + nbacth_files, + num_concentration_host, + state_scenario_host); + + real_type_2d_view_type num_concentration; + real_type_2d_view_type state; + + if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbacth_files is 1 + printf("-------------------------------------------------------\n"); + printf("--------------------Warning----------------------------\n"); + printf("Using cloned samples ... only for numerical experiments\n"); + state = real_type_2d_view_type("StateVector Devices", nBatch, stateVecDim); + state_host = Kokkos::create_mirror_view(state); + auto state_scenario_host_at_0 = Kokkos::subview(state_scenario_host, 0, Kokkos::ALL); + auto state_at_0 = Kokkos::subview(state, 0, Kokkos::ALL); + Kokkos::deep_copy(state_at_0, state_scenario_host_at_0); + TChem::Test::cloneView(state); + //make sure to copy state to host view. + Kokkos::deep_copy(state_host, state); + //scenario particles + auto num_concentration_host_at_0 = Kokkos::subview(num_concentration_host, 0, Kokkos::ALL); + num_concentration = real_type_2d_view_type("num_concentration", nBatch, amd.nParticles_); + auto num_concentration_at_0 = Kokkos::subview(num_concentration, 0, Kokkos::ALL); + Kokkos::deep_copy(num_concentration_at_0, num_concentration_host_at_0); + TChem::Test::cloneView(num_concentration); + + } else { + nBatch = nbacth_files; + state = real_type_2d_view_type("StateVector Devices", nBatch, stateVecDim); + Kokkos::deep_copy(state, state_scenario_host); + state_host = state_scenario_host; + + // scenario particles + num_concentration = real_type_2d_view_type("num_concentration", nBatch, amd.nParticles_); + Kokkos::deep_copy(num_concentration, num_concentration_host); + } + // output + real_type_2d_view_type rhs("rhs", nBatch, number_of_equations); + real_type_2d_view_type fac("fac", nBatch, number_of_equations); + real_type_3d_view_type jacobian("jacobian", nBatch, number_of_equations, number_of_equations); + + using policy_type = typename TChem::UseThisTeamPolicy::type; + + // fprintf(fout_times, " Computation type, total time [sec], time per sample [sec/sample]\n "); + + policy_type policy(exec_space_instance, nBatch, Kokkos::AUTO()); + + if (team_size > 0 && vector_size > 0) { + policy = policy_type(exec_space_instance, nBatch, team_size, vector_size); + } else if (team_size > 0 && vector_size < 0) { + // only set team size + policy = policy_type(exec_space_instance, nBatch, team_size); + } + + using range_type = Kokkos::pair; + const ordinal_type level = 1; + const ordinal_type per_team_extent = problem_type::getWorkSpaceSize(kmcd,amcd) + + number_of_equations; + const ordinal_type per_team_scratch = + TChem::Scratch::shmem_size(per_team_extent); + policy.set_scratch_size(level, Kokkos::PerTeam(per_team_scratch)); + + Kokkos::Timer timer; + FILE *fout_times = fopen(outputFileTimes.c_str(), "w"); + { + fprintf(fout_times, "{\n"); + fprintf(fout_times, " \"Aerosol RHSs\": \n {\n"); + const ordinal_type level = 1; + const std::string profile_name = "TChem::AerosolChemistry::RHS_evaluation"; + Kokkos::fence(); + timer.reset(); + Kokkos::Profiling::pushRegion(profile_name); + Kokkos::parallel_for + (profile_name, + policy, + KOKKOS_LAMBDA(const typename policy_type::member_type& member) { + const ordinal_type i = member.league_rank(); + + const ordinal_type m = problem_type::getNumberOfTimeODEs(kmcd,amcd); + const real_type_1d_view_type rhs_at_i = + Kokkos::subview(rhs, i, Kokkos::ALL()); + + const real_type_1d_view_type state_at_i = + Kokkos::subview(state, i, Kokkos::ALL()); + + const real_type_1d_view_type number_conc_at_i = + Kokkos::subview(num_concentration, i, Kokkos::ALL()); + TChem::Scratch work(member.team_scratch(level), + per_team_extent); + auto wptr = work.data(); + const ordinal_type total_n_species = kmcd.nSpec + amcd.nParticles*amcd.nSpec; + TChem::Impl::StateVector sv_at_i(total_n_species, state_at_i); + + const real_type temperature = sv_at_i.Temperature(); + const real_type pressure = sv_at_i.Pressure(); + const real_type_1d_view_type Ys = sv_at_i.MassFractions(); + const ordinal_type n_active_gas_species = kmcd.nSpec - kmcd.nConstSpec; + const auto activeYs = real_type_1d_view_type(Ys.data(), + n_active_gas_species ); + const auto constYs = real_type_1d_view_type(Ys.data(), + +n_active_gas_species, kmcd.nSpec ); + const real_type_1d_view_type partYs = Kokkos::subview(Ys, range_type(kmcd.nSpec, total_n_species)); + + real_type_1d_view_type vals(wptr, m); + wptr += m; + + /// problem workspace + /// problem setup + const ordinal_type problem_workspace_size = problem_type::getWorkSpaceSize(kmcd,amcd); + auto pw = real_type_1d_view_type(wptr, problem_workspace_size); + wptr +=problem_workspace_size; + problem_type problem; + problem._kmcd = kmcd; + problem._amcd = amcd; + + /// initialize problem + // problem._fac = fac_at_i; + problem._work = pw; + problem._temperature= temperature; + problem._pressure =pressure; + problem._const_concentration= constYs; + problem._number_conc =number_conc_at_i; + // active gas species + for (ordinal_type i=0;i work(member.team_scratch(level), + per_team_extent); + auto wptr = work.data(); + const ordinal_type total_n_species = kmcd.nSpec + amcd.nParticles*amcd.nSpec; + TChem::Impl::StateVector sv_at_i(total_n_species, state_at_i); + + const real_type temperature = sv_at_i.Temperature(); + const real_type pressure = sv_at_i.Pressure(); + const real_type_1d_view_type Ys = sv_at_i.MassFractions(); + const ordinal_type n_active_gas_species = kmcd.nSpec - kmcd.nConstSpec; + const auto activeYs = real_type_1d_view_type(Ys.data(), + n_active_gas_species ); + const auto constYs = real_type_1d_view_type(Ys.data(), + +n_active_gas_species, kmcd.nSpec ); + const real_type_1d_view_type partYs = Kokkos::subview(Ys, range_type(kmcd.nSpec, total_n_species)); + + real_type_1d_view_type vals(wptr, m); + wptr += m; + + /// problem workspace + /// problem setup + const ordinal_type problem_workspace_size = problem_type::getWorkSpaceSize(kmcd,amcd); + auto pw = real_type_1d_view_type(wptr, problem_workspace_size); + wptr +=problem_workspace_size; + problem_type problem; + problem._kmcd = kmcd; + problem._amcd = amcd; + + /// initialize problem + problem._fac = fac_at_i; + problem._work = pw; + problem._temperature= temperature; + problem._pressure =pressure; + problem._const_concentration= constYs; + problem._number_conc =number_conc_at_i; + // active gas species + for (ordinal_type i=0;i(0, nBatch), KOKKOS_LAMBDA(const ordinal_type &i) { printf("Devices::Initial condition sample No %d\n", i); @@ -139,7 +140,7 @@ int main(int argc, char *argv[]) { printf(" %e", state_at_i(k)); printf("\n"); }); - + } // auto writeState = [](const ordinal_type iter, const real_type_1d_view_host _t, const real_type_1d_view_host _dt, @@ -157,13 +158,13 @@ int main(int argc, char *argv[]) { }; - auto printState = [](const time_advance_type _tadv, const real_type _t, const real_type_1d_view_host _state_at_i) { - /// iter, t, dt, rho, pres, temp, Ys .... + auto printState = [](const time_advance_type _tadv, const real_type _t, const real_type_1d_view_host _state_at_i) + { printf("%e %e %e %e %e", _t, _t - _tadv._tbeg, _state_at_i(0), _state_at_i(1), _state_at_i(2)); for (ordinal_type k = 3, kend = _state_at_i.extent(0); k < kend; ++k) printf(" %e", _state_at_i(k)); printf("\n"); - + /// iter, t, dt, rho, pres, temp, Ys ... }; timer.reset(); @@ -253,6 +254,7 @@ int main(int argc, char *argv[]) { ordinal_type iter = 0; /// print of store QOI for the first sample + if (verbose) { /// could use cuda streams Kokkos::deep_copy(tadv_at_i_host, tadv_at_i); @@ -278,7 +280,7 @@ int main(int argc, char *argv[]) { TChem::AtmosphericChemistry::runDeviceBatch(policy, tol_newton, tol_time, fac, tadv, state, t, dt, state, kmcd); - { + if (verbose) { /// could use cuda streams Kokkos::deep_copy(tadv_at_i_host, tadv_at_i); Kokkos::deep_copy(t_at_i_host, t_at_i); @@ -314,6 +316,9 @@ int main(int argc, char *argv[]) { const real_type t_device_batch = timer.seconds(); printf("Time ignition %e [sec] %e [sec/sample]\n", t_device_batch, t_device_batch / real_type(nBatch)); + + if (verbose) + { Kokkos::parallel_for( Kokkos::RangePolicy(0, nBatch), KOKKOS_LAMBDA(const ordinal_type &i) { printf("Devices:: Solution sample No %d\n", i); @@ -322,6 +327,7 @@ int main(int argc, char *argv[]) { printf(" %e", state_at_i(k)); printf("\n"); }); + } fclose(fout); From 37c513681ba44e17f781b60711064e83834c3dc9 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 9 Sep 2024 16:33:15 -0600 Subject: [PATCH 02/19] Add routine to modify the number of particles from AMD. Use this new feature to modify the number of particles in the RHS and Jacobian example. --- src/core/TChem_AerosolModelData.cpp | 11 +++++++++++ src/core/TChem_AerosolModelData.hpp | 4 +++- src/examples/TChem_AerosolChemistry_RHSs.cpp | 6 +++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/core/TChem_AerosolModelData.cpp b/src/core/TChem_AerosolModelData.cpp index 59e0e9ec..a8e6fde3 100644 --- a/src/core/TChem_AerosolModelData.cpp +++ b/src/core/TChem_AerosolModelData.cpp @@ -225,6 +225,17 @@ int AerosolModelData::initChem(YAML::Node &root, return 0; } +void AerosolModelData::setNumberofParticles(const ordinal_type number_of_particles) +{ + printf("-------------------------------------------------------\n"); + printf("--------------------Warning----------------------------\n"); + printf("Setting number of particles\n"); + printf("Old value : %d \n", nParticles_); + nParticles_=number_of_particles; + printf("Current value : %d \n", nParticles_); + printf("-------------------------------------------------------\n"); +} + void AerosolModelData::scenarioConditionParticles(const std::string &mechfile, const ordinal_type nBatch, real_type_2d_view_host& num_concentration, diff --git a/src/core/TChem_AerosolModelData.hpp b/src/core/TChem_AerosolModelData.hpp index 3cc9ded8..6e6678e0 100644 --- a/src/core/TChem_AerosolModelData.hpp +++ b/src/core/TChem_AerosolModelData.hpp @@ -37,7 +37,7 @@ namespace TChem { ordinal_type nSpec_gas_; ordinal_type nConstSpec_gas_; // number of particles - ordinal_type nParticles_; + ordinal_type nParticles_{-1}; // aerosol molecular weights and density real_type_1d_dual_view molecular_weights_, aerosol_density_; simpol_phase_transfer_type_1d_dual_view simpol_params_; @@ -64,6 +64,8 @@ namespace TChem { void initFile(const std::string &mechfile, std::ostream& echofile); ordinal_type initChem(YAML::Node& doc, std::ostream& echofile); + void setNumberofParticles(const ordinal_type number_of_particles); + void setGasParameters(const KineticModelData& kmd); void scenarioConditionParticles(const std::string &mechfile, const ordinal_type nBatch, diff --git a/src/examples/TChem_AerosolChemistry_RHSs.cpp b/src/examples/TChem_AerosolChemistry_RHSs.cpp index 2c013d6f..ae773700 100644 --- a/src/examples/TChem_AerosolChemistry_RHSs.cpp +++ b/src/examples/TChem_AerosolChemistry_RHSs.cpp @@ -36,6 +36,7 @@ int main(int argc, char *argv[]) { int nBatch(1), team_size(-1), vector_size(-1); bool verbose(true); bool use_cloned_samples(false); + int number_of_particles(-1); /// parse command line arguments TChem::CommandLineParser opts("This example computes the net production rates with a given state vector"); @@ -51,6 +52,7 @@ int main(int argc, char *argv[]) { opts.set_option("outputfile_times", "Wal times file name e.g., times.json", &outputFileTimes); opts.set_option("team_thread_size", "time thread size ", &team_size); opts.set_option("vector_thread_size", "vector thread size ", &vector_size); + opts.set_option("number_of_particles", "Set the number of particles; this will overwrite the value from the input file. ", &number_of_particles); opts.set_option("verbose", "If true, printout the first omega values", &verbose); opts.set_option("batch_size", " number of batches or samples, e.g. 10 ", &nBatch); opts.set_option( @@ -98,6 +100,9 @@ int main(int argc, char *argv[]) { printf("amd parsing ...\n"); TChem::AerosolModelData amd(aeroFile, kmd); + if(number_of_particles > 0) { + amd.setNumberofParticles(number_of_particles); + } const auto amcd = TChem::create_AerosolModelConstData(amd); const ordinal_type total_n_species =kmcd.nSpec + amcd.nSpec * amcd.nParticles; @@ -275,7 +280,6 @@ int main(int argc, char *argv[]) { { const ordinal_type level = 1; - fprintf(fout_times, "{\n"); fprintf(fout_times, " \"Aerosol Numerical Jacobian\": \n {\n"); const std::string profile_name = "TChem::AerosolChemistry::NumericalJacobian_evaluation"; Kokkos::fence(); From 52d2588f8f0f263c8985bcc67d72bed252531b41 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 12 Sep 2024 14:51:09 -0600 Subject: [PATCH 03/19] Scripts for wall times. --- .../CB05CL_AE5_w_simpolSOA/inputs.sh | 8 ++ aerosol_performance/runAll.sh | 125 ++++++++++++++++++ aerosol_performance/runThisCPU.sh | 31 +++++ aerosol_performance/runThisGPU.sh | 33 +++++ 4 files changed, 197 insertions(+) create mode 100755 aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh create mode 100755 aerosol_performance/runAll.sh create mode 100644 aerosol_performance/runThisCPU.sh create mode 100644 aerosol_performance/runThisGPU.sh diff --git a/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh b/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh new file mode 100755 index 00000000..4c2ae18c --- /dev/null +++ b/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh @@ -0,0 +1,8 @@ +inputs=${TCHEM_INSTALL_PATH}/examples/runs/atmospheric_chemistry/CB05CL_AE5_w_simpolSOA +inputfile_particles=${inputs}/scenario_conditions_particle.yaml +chemfile=${inputs}/config_full_gas.yaml +aerofile=${inputs}/mechanism_aero.yaml + +export scenario_n_inputs="--chemfile=$chemfile \ + --aerofile=$aerofile \ + --inputfile_particles=$inputfile_particles" diff --git a/aerosol_performance/runAll.sh b/aerosol_performance/runAll.sh new file mode 100755 index 00000000..269ca8bf --- /dev/null +++ b/aerosol_performance/runAll.sh @@ -0,0 +1,125 @@ +#measure wall times for the following mechanisms +dirs=(CB05CL_AE5_w_simpolSOA) +sacado_flags=(no_sacado) +export experiment_base=TChem_AerosolChemistry + +cpu_solver_strList=( + # "trbdf" + # "kokkoskernels" + # "cvode" + "rhss" +) +cpu_exe_string=( + # "" + # "_KokkosKernels" + # "_CVODE" + "_RHSs" +) +gpu_solver_strList=( + # "trbdf" + # "kokkoskernels" + #"rhss" +) +gpu_exe_string=( + # "" + # "_KokkosKernels" + #"_RHSs" +) + +rhss_params(){ + export numerical_params="" +} +trbdf_params(){ + t_iterPerInt=1 + min_dt='1e-3' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --atol-time=${atol_t} \ + --tend=$tend \ + --atol-newton=1e-18 \ + --rtol-newton=1e-8 \ + --max-newton-iterations=20 \ + --max-time-iterations=20000 " + +} + +kokkoskernels_params(){ + t_iterPerInt=1 + min_dt='1e-1' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --atol-time=${atol_t} \ + --tend=$tend " +} + +cvode_params(){ + t_iterPerInt=10 + min_dt='1e-20' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --use-cvode=true \ + --atol-time=${atol_t} \ + --tend=$tend " +} + +exe=runHOSTWS.sh +for i in "${!cpu_solver_strList[@]}"; do + export experiment_name="${cpu_solver_strList[i]}" + export experiment_suffix="${cpu_exe_string[i]}" + params_fxn="${experiment_name}_params" + printf "setting variables for %s case (CPU)\n" "${experiment_name}" + ${params_fxn} + for dir in ${dirs[@]}; do + for sacado in ${sacado_flags[@]}; do + if [ "${experiment_name}" == "expEuler" ] && [ "${sacado}" == "sacado" ]; then + continue + elif [ "${experiment_name}" == "cvode" ] && [ "${sacado}" == "sacado" ]; then + continue + fi + export sflag=$sacado + run_this="cd $dir;./$exe;cd -" + echo $run_this + eval $run_this + done + done + echo "end of outer loop" +done + +exe=runCUDA.sh +for i in "${!gpu_solver_strList[@]}"; do + export experiment_name="${gpu_solver_strList[i]}" + export experiment_suffix="${gpu_exe_string[i]}" + params_fxn="${experiment_name}_params" + printf "setting variables for %s case (GPU)\n" "${experiment_name}" + ${params_fxn} + for dir in ${dirs[@]}; do + for sacado in ${sacado_flags[@]}; do + if [ "${experiment_name}" == "expEuler" ] && [ "${sacado}" == "sacado" ]; then + continue + fi + export sflag=$sacado + run_this="cd $dir;./$exe;cd -" + echo $run_this + eval $run_this + done + done + echo "end of outer loop" +done diff --git a/aerosol_performance/runThisCPU.sh b/aerosol_performance/runThisCPU.sh new file mode 100644 index 00000000..e8709cff --- /dev/null +++ b/aerosol_performance/runThisCPU.sh @@ -0,0 +1,31 @@ +for number_of_particles in ${nparticles[@]}; do +for N in ${nbatch[@]}; do + for threads in ${Nthread[@]}; do + echo "nbatch = $N" + echo "Nthread = $threads" + thread_size=${threads} + output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" + output="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" + echo "${tchem_outputs}" + echo "${experiment_name}" + echo "/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" + echo "output file = " $output + run_this="OMP_NUM_THREADS=$thread_size OMP_PLACES=threads OMP_PROC_BIND=close ${exec} \ + --batch_size=$N \ + --use_cloned_samples=$use_cloned_samples \ + --verbose=$verbose \ + --outputfile_times=$output_wall_times \ + --outputfile=$output \ + --number_of_particles=$number_of_particles \ + $scenario_n_inputs \ + $numerical_params " + echo $run_this + eval $run_this + sleep 2 + # Note: we use this with kokkos tools + kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" + sleep 2 + rm -rf $machine_name* + done +done +done diff --git a/aerosol_performance/runThisGPU.sh b/aerosol_performance/runThisGPU.sh new file mode 100644 index 00000000..7f25d88b --- /dev/null +++ b/aerosol_performance/runThisGPU.sh @@ -0,0 +1,33 @@ +for number_of_particles in ${nparticles[@]}; do +for N in ${nbatch[@]}; do + for i in "${!vector_size[@]}"; do + vector_thread_size=${vector_size[i]} + team_thread_size=${team_size[i]} + echo "nbatch = $N" + echo "vector thread size = $vector_thread_size" + echo "team thread size = $team_thread_size" + output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.json" + output_file="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" + echo "${tchem_outputs}" + echo "${experiment_name}" + echo "/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" + echo "output file = ${output_file}" + run_this="OMP_NUM_THREADS=1 OMP_PLACES=threads OMP_PROC_BIND=close $exec \ + --batch_size=$N \ + --team_thread_size=$vector_thread_size \ + --vector_thread_size=$vector_thread_size \ + --use_cloned_samples=$use_cloned_samples \ + --verbose=$verbose \ + --outputfile_times=$output_wall_times \ + --outputfile=$output_file \ + $scenario_n_inputs \ + $numerical_params " + echo $run_this + eval $run_this + sleep 2 + kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.json" + sleep 2 + rm -rf $machine_name* +done +done +done From 9f65694d617a9472499d16728219d33ee30445f0 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 12 Sep 2024 14:54:44 -0600 Subject: [PATCH 04/19] Add machine file template. --- aerosol_performance/loadMachinePathsTemplate.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 aerosol_performance/loadMachinePathsTemplate.sh diff --git a/aerosol_performance/loadMachinePathsTemplate.sh b/aerosol_performance/loadMachinePathsTemplate.sh new file mode 100644 index 00000000..f009fca5 --- /dev/null +++ b/aerosol_performance/loadMachinePathsTemplate.sh @@ -0,0 +1,6 @@ + +export KOKKOS_TOOLS_LIBS=/path/kokkostools/libs/libkp_kernel_timer.so +TCHEM_INSTALL_PATH=/path/tchem/install +exec=${TCHEM_INSTALL_PATH}/examples/${experiment_exe_root}.x +machine_name=/name/machine + From 89e4c540d40d06a0f98b57a4c2eee0f3fea459f5 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 12 Sep 2024 16:14:00 -0600 Subject: [PATCH 05/19] script to build/install/clone kokkostools --- scripts/build_kokkostools.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 scripts/build_kokkostools.sh diff --git a/scripts/build_kokkostools.sh b/scripts/build_kokkostools.sh new file mode 100755 index 00000000..9231291a --- /dev/null +++ b/scripts/build_kokkostools.sh @@ -0,0 +1,34 @@ + +MY_CC=gcc +MY_CXX=g++ +JFLAG="-j 10" + +BUILD_BASE=${PWD} +INSTALL_BASE=${PWD}/install + +get_kokkostools (){ +echo "get kokkostools:" +if [ -d "${KOKKOSTOOLS_REPOSITORY_PATH}" ] && [ "$(ls -A ${KOKKOSTOOLS_REPOSITORY_PATH})" ]; then + echo "${KOKKOSTOOLS_REPOSITORY_PATH} exists and is not empty ... aborting clone"; return +fi +git clone https://github.com/kokkos/kokkos-tools.git ${KOKKOSTOOLS_REPOSITORY_PATH} +} + +build_install_kokkostools(){ +echo "Building kokkos tools:" +mkdir ${KOKKOSTOOLS_BUILD_PATH} +cd ${KOKKOSTOOLS_BUILD_PATH} +cmake \ + -D CMAKE_INSTALL_PREFIX=${KOKKOSTOOLS_INSTALL_PATH} \ + -D CMAKE_CXX_COMPILER="${MY_CXX}" \ + -D CMAKE_C_COMPILER="${MY_CC}" \ + -D CMAKE_BUILD_TYPE=RELEASE \ + ${KOKKOSTOOLS_REPOSITORY_PATH} +make ${JFLAG} install +} +KOKKOSTOOLS_REPOSITORY_PATH=/ascldap/users/odiazib/Documents/Oscar/TChem/kokkos-tools +KOKKOSTOOLS_BUILD_PATH=${BUILD_BASE}/kokkos-tools +KOKKOSTOOLS_INSTALL_PATH=${INSTALL_BASE}/kokkos-tools +build_install_kokkostools + +exit From 2004c243f1ecdee46d1aeff2b63abc829a4b0bfa Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 12 Sep 2024 16:23:06 -0600 Subject: [PATCH 06/19] Move to script folder. --- .../CB05CL_AE5_w_simpolSOA/inputs.sh | 8 ++ .../loadMachinePathsTemplate.sh | 6 + scripts/aerosol_performance/runAll.sh | 125 ++++++++++++++++++ scripts/aerosol_performance/runThisCPU.sh | 31 +++++ scripts/aerosol_performance/runThisGPU.sh | 33 +++++ 5 files changed, 203 insertions(+) create mode 100755 scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh create mode 100644 scripts/aerosol_performance/loadMachinePathsTemplate.sh create mode 100755 scripts/aerosol_performance/runAll.sh create mode 100644 scripts/aerosol_performance/runThisCPU.sh create mode 100644 scripts/aerosol_performance/runThisGPU.sh diff --git a/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh new file mode 100755 index 00000000..4c2ae18c --- /dev/null +++ b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh @@ -0,0 +1,8 @@ +inputs=${TCHEM_INSTALL_PATH}/examples/runs/atmospheric_chemistry/CB05CL_AE5_w_simpolSOA +inputfile_particles=${inputs}/scenario_conditions_particle.yaml +chemfile=${inputs}/config_full_gas.yaml +aerofile=${inputs}/mechanism_aero.yaml + +export scenario_n_inputs="--chemfile=$chemfile \ + --aerofile=$aerofile \ + --inputfile_particles=$inputfile_particles" diff --git a/scripts/aerosol_performance/loadMachinePathsTemplate.sh b/scripts/aerosol_performance/loadMachinePathsTemplate.sh new file mode 100644 index 00000000..f009fca5 --- /dev/null +++ b/scripts/aerosol_performance/loadMachinePathsTemplate.sh @@ -0,0 +1,6 @@ + +export KOKKOS_TOOLS_LIBS=/path/kokkostools/libs/libkp_kernel_timer.so +TCHEM_INSTALL_PATH=/path/tchem/install +exec=${TCHEM_INSTALL_PATH}/examples/${experiment_exe_root}.x +machine_name=/name/machine + diff --git a/scripts/aerosol_performance/runAll.sh b/scripts/aerosol_performance/runAll.sh new file mode 100755 index 00000000..269ca8bf --- /dev/null +++ b/scripts/aerosol_performance/runAll.sh @@ -0,0 +1,125 @@ +#measure wall times for the following mechanisms +dirs=(CB05CL_AE5_w_simpolSOA) +sacado_flags=(no_sacado) +export experiment_base=TChem_AerosolChemistry + +cpu_solver_strList=( + # "trbdf" + # "kokkoskernels" + # "cvode" + "rhss" +) +cpu_exe_string=( + # "" + # "_KokkosKernels" + # "_CVODE" + "_RHSs" +) +gpu_solver_strList=( + # "trbdf" + # "kokkoskernels" + #"rhss" +) +gpu_exe_string=( + # "" + # "_KokkosKernels" + #"_RHSs" +) + +rhss_params(){ + export numerical_params="" +} +trbdf_params(){ + t_iterPerInt=1 + min_dt='1e-3' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --atol-time=${atol_t} \ + --tend=$tend \ + --atol-newton=1e-18 \ + --rtol-newton=1e-8 \ + --max-newton-iterations=20 \ + --max-time-iterations=20000 " + +} + +kokkoskernels_params(){ + t_iterPerInt=1 + min_dt='1e-1' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --atol-time=${atol_t} \ + --tend=$tend " +} + +cvode_params(){ + t_iterPerInt=10 + min_dt='1e-20' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --use-cvode=true \ + --atol-time=${atol_t} \ + --tend=$tend " +} + +exe=runHOSTWS.sh +for i in "${!cpu_solver_strList[@]}"; do + export experiment_name="${cpu_solver_strList[i]}" + export experiment_suffix="${cpu_exe_string[i]}" + params_fxn="${experiment_name}_params" + printf "setting variables for %s case (CPU)\n" "${experiment_name}" + ${params_fxn} + for dir in ${dirs[@]}; do + for sacado in ${sacado_flags[@]}; do + if [ "${experiment_name}" == "expEuler" ] && [ "${sacado}" == "sacado" ]; then + continue + elif [ "${experiment_name}" == "cvode" ] && [ "${sacado}" == "sacado" ]; then + continue + fi + export sflag=$sacado + run_this="cd $dir;./$exe;cd -" + echo $run_this + eval $run_this + done + done + echo "end of outer loop" +done + +exe=runCUDA.sh +for i in "${!gpu_solver_strList[@]}"; do + export experiment_name="${gpu_solver_strList[i]}" + export experiment_suffix="${gpu_exe_string[i]}" + params_fxn="${experiment_name}_params" + printf "setting variables for %s case (GPU)\n" "${experiment_name}" + ${params_fxn} + for dir in ${dirs[@]}; do + for sacado in ${sacado_flags[@]}; do + if [ "${experiment_name}" == "expEuler" ] && [ "${sacado}" == "sacado" ]; then + continue + fi + export sflag=$sacado + run_this="cd $dir;./$exe;cd -" + echo $run_this + eval $run_this + done + done + echo "end of outer loop" +done diff --git a/scripts/aerosol_performance/runThisCPU.sh b/scripts/aerosol_performance/runThisCPU.sh new file mode 100644 index 00000000..e8709cff --- /dev/null +++ b/scripts/aerosol_performance/runThisCPU.sh @@ -0,0 +1,31 @@ +for number_of_particles in ${nparticles[@]}; do +for N in ${nbatch[@]}; do + for threads in ${Nthread[@]}; do + echo "nbatch = $N" + echo "Nthread = $threads" + thread_size=${threads} + output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" + output="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" + echo "${tchem_outputs}" + echo "${experiment_name}" + echo "/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" + echo "output file = " $output + run_this="OMP_NUM_THREADS=$thread_size OMP_PLACES=threads OMP_PROC_BIND=close ${exec} \ + --batch_size=$N \ + --use_cloned_samples=$use_cloned_samples \ + --verbose=$verbose \ + --outputfile_times=$output_wall_times \ + --outputfile=$output \ + --number_of_particles=$number_of_particles \ + $scenario_n_inputs \ + $numerical_params " + echo $run_this + eval $run_this + sleep 2 + # Note: we use this with kokkos tools + kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" + sleep 2 + rm -rf $machine_name* + done +done +done diff --git a/scripts/aerosol_performance/runThisGPU.sh b/scripts/aerosol_performance/runThisGPU.sh new file mode 100644 index 00000000..7f25d88b --- /dev/null +++ b/scripts/aerosol_performance/runThisGPU.sh @@ -0,0 +1,33 @@ +for number_of_particles in ${nparticles[@]}; do +for N in ${nbatch[@]}; do + for i in "${!vector_size[@]}"; do + vector_thread_size=${vector_size[i]} + team_thread_size=${team_size[i]} + echo "nbatch = $N" + echo "vector thread size = $vector_thread_size" + echo "team thread size = $team_thread_size" + output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.json" + output_file="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" + echo "${tchem_outputs}" + echo "${experiment_name}" + echo "/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" + echo "output file = ${output_file}" + run_this="OMP_NUM_THREADS=1 OMP_PLACES=threads OMP_PROC_BIND=close $exec \ + --batch_size=$N \ + --team_thread_size=$vector_thread_size \ + --vector_thread_size=$vector_thread_size \ + --use_cloned_samples=$use_cloned_samples \ + --verbose=$verbose \ + --outputfile_times=$output_wall_times \ + --outputfile=$output_file \ + $scenario_n_inputs \ + $numerical_params " + echo $run_this + eval $run_this + sleep 2 + kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.json" + sleep 2 + rm -rf $machine_name* +done +done +done From b91c0c749d95cbcd24ee87ec2f6e749cb15b02d0 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 12 Sep 2024 16:24:31 -0600 Subject: [PATCH 07/19] Remove files. --- .../CB05CL_AE5_w_simpolSOA/inputs.sh | 8 -- .../loadMachinePathsTemplate.sh | 6 - aerosol_performance/runAll.sh | 125 ------------------ aerosol_performance/runThisCPU.sh | 31 ----- aerosol_performance/runThisGPU.sh | 33 ----- 5 files changed, 203 deletions(-) delete mode 100755 aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh delete mode 100644 aerosol_performance/loadMachinePathsTemplate.sh delete mode 100755 aerosol_performance/runAll.sh delete mode 100644 aerosol_performance/runThisCPU.sh delete mode 100644 aerosol_performance/runThisGPU.sh diff --git a/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh b/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh deleted file mode 100755 index 4c2ae18c..00000000 --- a/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh +++ /dev/null @@ -1,8 +0,0 @@ -inputs=${TCHEM_INSTALL_PATH}/examples/runs/atmospheric_chemistry/CB05CL_AE5_w_simpolSOA -inputfile_particles=${inputs}/scenario_conditions_particle.yaml -chemfile=${inputs}/config_full_gas.yaml -aerofile=${inputs}/mechanism_aero.yaml - -export scenario_n_inputs="--chemfile=$chemfile \ - --aerofile=$aerofile \ - --inputfile_particles=$inputfile_particles" diff --git a/aerosol_performance/loadMachinePathsTemplate.sh b/aerosol_performance/loadMachinePathsTemplate.sh deleted file mode 100644 index f009fca5..00000000 --- a/aerosol_performance/loadMachinePathsTemplate.sh +++ /dev/null @@ -1,6 +0,0 @@ - -export KOKKOS_TOOLS_LIBS=/path/kokkostools/libs/libkp_kernel_timer.so -TCHEM_INSTALL_PATH=/path/tchem/install -exec=${TCHEM_INSTALL_PATH}/examples/${experiment_exe_root}.x -machine_name=/name/machine - diff --git a/aerosol_performance/runAll.sh b/aerosol_performance/runAll.sh deleted file mode 100755 index 269ca8bf..00000000 --- a/aerosol_performance/runAll.sh +++ /dev/null @@ -1,125 +0,0 @@ -#measure wall times for the following mechanisms -dirs=(CB05CL_AE5_w_simpolSOA) -sacado_flags=(no_sacado) -export experiment_base=TChem_AerosolChemistry - -cpu_solver_strList=( - # "trbdf" - # "kokkoskernels" - # "cvode" - "rhss" -) -cpu_exe_string=( - # "" - # "_KokkosKernels" - # "_CVODE" - "_RHSs" -) -gpu_solver_strList=( - # "trbdf" - # "kokkoskernels" - #"rhss" -) -gpu_exe_string=( - # "" - # "_KokkosKernels" - #"_RHSs" -) - -rhss_params(){ - export numerical_params="" -} -trbdf_params(){ - t_iterPerInt=1 - min_dt='1e-3' - max_dt='1' - atol_t='1e-12' - tol_time='1e-3' - tend='1' - export numerical_params="--tol-time=$tol_time \ - --time-iterations-per-interval=$t_iterPerInt \ - --dtmin=$min_dt \ - --dtmax=$max_dt \ - --atol-time=${atol_t} \ - --tend=$tend \ - --atol-newton=1e-18 \ - --rtol-newton=1e-8 \ - --max-newton-iterations=20 \ - --max-time-iterations=20000 " - -} - -kokkoskernels_params(){ - t_iterPerInt=1 - min_dt='1e-1' - max_dt='1' - atol_t='1e-12' - tol_time='1e-3' - tend='1' - export numerical_params="--tol-time=$tol_time \ - --time-iterations-per-interval=$t_iterPerInt \ - --dtmin=$min_dt \ - --dtmax=$max_dt \ - --atol-time=${atol_t} \ - --tend=$tend " -} - -cvode_params(){ - t_iterPerInt=10 - min_dt='1e-20' - max_dt='1' - atol_t='1e-12' - tol_time='1e-3' - tend='1' - export numerical_params="--tol-time=$tol_time \ - --time-iterations-per-interval=$t_iterPerInt \ - --dtmin=$min_dt \ - --dtmax=$max_dt \ - --use-cvode=true \ - --atol-time=${atol_t} \ - --tend=$tend " -} - -exe=runHOSTWS.sh -for i in "${!cpu_solver_strList[@]}"; do - export experiment_name="${cpu_solver_strList[i]}" - export experiment_suffix="${cpu_exe_string[i]}" - params_fxn="${experiment_name}_params" - printf "setting variables for %s case (CPU)\n" "${experiment_name}" - ${params_fxn} - for dir in ${dirs[@]}; do - for sacado in ${sacado_flags[@]}; do - if [ "${experiment_name}" == "expEuler" ] && [ "${sacado}" == "sacado" ]; then - continue - elif [ "${experiment_name}" == "cvode" ] && [ "${sacado}" == "sacado" ]; then - continue - fi - export sflag=$sacado - run_this="cd $dir;./$exe;cd -" - echo $run_this - eval $run_this - done - done - echo "end of outer loop" -done - -exe=runCUDA.sh -for i in "${!gpu_solver_strList[@]}"; do - export experiment_name="${gpu_solver_strList[i]}" - export experiment_suffix="${gpu_exe_string[i]}" - params_fxn="${experiment_name}_params" - printf "setting variables for %s case (GPU)\n" "${experiment_name}" - ${params_fxn} - for dir in ${dirs[@]}; do - for sacado in ${sacado_flags[@]}; do - if [ "${experiment_name}" == "expEuler" ] && [ "${sacado}" == "sacado" ]; then - continue - fi - export sflag=$sacado - run_this="cd $dir;./$exe;cd -" - echo $run_this - eval $run_this - done - done - echo "end of outer loop" -done diff --git a/aerosol_performance/runThisCPU.sh b/aerosol_performance/runThisCPU.sh deleted file mode 100644 index e8709cff..00000000 --- a/aerosol_performance/runThisCPU.sh +++ /dev/null @@ -1,31 +0,0 @@ -for number_of_particles in ${nparticles[@]}; do -for N in ${nbatch[@]}; do - for threads in ${Nthread[@]}; do - echo "nbatch = $N" - echo "Nthread = $threads" - thread_size=${threads} - output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" - output="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" - echo "${tchem_outputs}" - echo "${experiment_name}" - echo "/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" - echo "output file = " $output - run_this="OMP_NUM_THREADS=$thread_size OMP_PLACES=threads OMP_PROC_BIND=close ${exec} \ - --batch_size=$N \ - --use_cloned_samples=$use_cloned_samples \ - --verbose=$verbose \ - --outputfile_times=$output_wall_times \ - --outputfile=$output \ - --number_of_particles=$number_of_particles \ - $scenario_n_inputs \ - $numerical_params " - echo $run_this - eval $run_this - sleep 2 - # Note: we use this with kokkos tools - kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" - sleep 2 - rm -rf $machine_name* - done -done -done diff --git a/aerosol_performance/runThisGPU.sh b/aerosol_performance/runThisGPU.sh deleted file mode 100644 index 7f25d88b..00000000 --- a/aerosol_performance/runThisGPU.sh +++ /dev/null @@ -1,33 +0,0 @@ -for number_of_particles in ${nparticles[@]}; do -for N in ${nbatch[@]}; do - for i in "${!vector_size[@]}"; do - vector_thread_size=${vector_size[i]} - team_thread_size=${team_size[i]} - echo "nbatch = $N" - echo "vector thread size = $vector_thread_size" - echo "team thread size = $team_thread_size" - output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.json" - output_file="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" - echo "${tchem_outputs}" - echo "${experiment_name}" - echo "/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" - echo "output file = ${output_file}" - run_this="OMP_NUM_THREADS=1 OMP_PLACES=threads OMP_PROC_BIND=close $exec \ - --batch_size=$N \ - --team_thread_size=$vector_thread_size \ - --vector_thread_size=$vector_thread_size \ - --use_cloned_samples=$use_cloned_samples \ - --verbose=$verbose \ - --outputfile_times=$output_wall_times \ - --outputfile=$output_file \ - $scenario_n_inputs \ - $numerical_params " - echo $run_this - eval $run_this - sleep 2 - kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.json" - sleep 2 - rm -rf $machine_name* -done -done -done From a1b68f7565773c35fbd99c6d98301e41e3ffd7bf Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 12 Sep 2024 16:39:43 -0600 Subject: [PATCH 08/19] Fixing paths --- scripts/build_kokkostools.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/build_kokkostools.sh b/scripts/build_kokkostools.sh index 9231291a..71255c09 100755 --- a/scripts/build_kokkostools.sh +++ b/scripts/build_kokkostools.sh @@ -3,8 +3,9 @@ MY_CC=gcc MY_CXX=g++ JFLAG="-j 10" -BUILD_BASE=${PWD} -INSTALL_BASE=${PWD}/install +REPO_BASE=${PWD}/kokkos-tools +BUILD_BASE=${PWD}/kokkos-tools +INSTALL_BASE=${PWD}/kokkos-tools get_kokkostools (){ echo "get kokkostools:" @@ -26,9 +27,11 @@ cmake \ ${KOKKOSTOOLS_REPOSITORY_PATH} make ${JFLAG} install } -KOKKOSTOOLS_REPOSITORY_PATH=/ascldap/users/odiazib/Documents/Oscar/TChem/kokkos-tools -KOKKOSTOOLS_BUILD_PATH=${BUILD_BASE}/kokkos-tools -KOKKOSTOOLS_INSTALL_PATH=${INSTALL_BASE}/kokkos-tools +KOKKOSTOOLS_REPOSITORY_PATH=${REPO_BASE}/main +KOKKOSTOOLS_BUILD_PATH=${BUILD_BASE}/build +KOKKOSTOOLS_INSTALL_PATH=${INSTALL_BASE}/install + +get_kokkostools build_install_kokkostools exit From 8c69f25d631e06d6f1d9546cfe75dc19a3f5d62d Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 12 Sep 2024 16:54:48 -0600 Subject: [PATCH 09/19] Missing scripts --- .../CB05CL_AE5_w_simpolSOA/runCUDA.sh | 23 ++++++++++++++++++ .../CB05CL_AE5_w_simpolSOA/runHOSTWS.sh | 24 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runCUDA.sh create mode 100755 scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runHOSTWS.sh diff --git a/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runCUDA.sh b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runCUDA.sh new file mode 100755 index 00000000..c4df4714 --- /dev/null +++ b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runCUDA.sh @@ -0,0 +1,23 @@ +export DEVICE=GPU +export experiment_exe_root=${experiment_base}${experiment_suffix} +export sacado_flag=$sflag +echo "sacado flag = ${sacado_flag}" +source ../loadGPU_WS.sh +source ./inputs.sh + +export experiment_name="${experiment_name}-${sacado_flag}" +echo "experiment name is ${experiment_name}" +echo "experiment exe is ${exec}" +use_cloned_samples=true +# change to true if you want to output the reaction rates outputs. +verbose=false +#we will save outputs in this directory +tchem_outputs=CUDA +mkdir -p ${tchem_outputs}/${experiment_name} +nbatch=(72 144 216) +# 288 360 432 504 576 648 720 792 864 936 1008) +# let's use for now 1 and 1 for team and vector size; however, we need to tune up these parameters. +vector_size=(1) +team_size=(1) +nparticles=(1 10 100 1000) +source ../runThisGPU.sh diff --git a/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runHOSTWS.sh b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runHOSTWS.sh new file mode 100755 index 00000000..17dd665a --- /dev/null +++ b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runHOSTWS.sh @@ -0,0 +1,24 @@ +export DEVICE=CPU +export experiment_exe_root=${experiment_base}${experiment_suffix} +export sacado_flag=$sflag +echo "sacado flag = ${sacado_flag}" +source ../loadCPU_WS.sh +source ./inputs.sh + +export experiment_name="${experiment_name}-${sacado_flag}" +echo "experiment name is ${experiment_name}" +echo "experiment exe is ${exec}" +use_cloned_samples=true +# change to true if you want to output the reaction rates outputs. +verbose=false +#we will save outputs in this directory +tchem_outputs=HOST +mkdir -p ${tchem_outputs}/${experiment_name} +#20 100 1000 10000 100000 200000 500000 1000000 2000000 3000000 4500000 +nbatch=(1) +# 144 216 288) +# 360 432 504 576 648 720 792 864 936 1008) +Nthread=(52 104) +nparticles=(1 10 50 100) +# 500 1000 5000 10000 +source ../runThisCPU.sh From 79e1ed0fb3f2317d3c7380ba027e8fdbe70bc68c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Thu, 12 Sep 2024 17:27:54 -0600 Subject: [PATCH 10/19] Fixing file names. --- scripts/aerosol_performance/runThisCPU.sh | 2 +- scripts/aerosol_performance/runThisGPU.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/aerosol_performance/runThisCPU.sh b/scripts/aerosol_performance/runThisCPU.sh index e8709cff..f554ad62 100644 --- a/scripts/aerosol_performance/runThisCPU.sh +++ b/scripts/aerosol_performance/runThisCPU.sh @@ -5,7 +5,7 @@ for N in ${nbatch[@]}; do echo "Nthread = $threads" thread_size=${threads} output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" - output="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" + output="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.txt" echo "${tchem_outputs}" echo "${experiment_name}" echo "/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" diff --git a/scripts/aerosol_performance/runThisGPU.sh b/scripts/aerosol_performance/runThisGPU.sh index 7f25d88b..bc9654a0 100644 --- a/scripts/aerosol_performance/runThisGPU.sh +++ b/scripts/aerosol_performance/runThisGPU.sh @@ -6,8 +6,8 @@ for N in ${nbatch[@]}; do echo "nbatch = $N" echo "vector thread size = $vector_thread_size" echo "team thread size = $team_thread_size" - output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.json" - output_file="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" + output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}_number_of_particles${number_of_particles}.json" + output_file="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}_number_of_particles${number_of_particles}.txt" echo "${tchem_outputs}" echo "${experiment_name}" echo "/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" @@ -25,7 +25,7 @@ for N in ${nbatch[@]}; do echo $run_this eval $run_this sleep 2 - kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.json" + kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}_number_of_particles${number_of_particles}.json" sleep 2 rm -rf $machine_name* done From 80f14a5bd40c1cdf36cc18c00ca439868d7024f8 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Fri, 13 Sep 2024 18:34:05 -0600 Subject: [PATCH 11/19] Using ThreadVectorRange and removing team_barrier. --- src/core/impl/TChem_Impl_Aerosol_RHS.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/impl/TChem_Impl_Aerosol_RHS.hpp b/src/core/impl/TChem_Impl_Aerosol_RHS.hpp index 9fa67ba1..ba834da6 100644 --- a/src/core/impl/TChem_Impl_Aerosol_RHS.hpp +++ b/src/core/impl/TChem_Impl_Aerosol_RHS.hpp @@ -84,10 +84,10 @@ struct Aerosol_RHS work, kmcd); // 2. update RHS of gas and aerosol species - member.team_barrier(); + // member.team_barrier(); using SIMPOL_single_particle_type = TChem::Impl::SIMPOL_single_particle; Kokkos::parallel_for( - Kokkos::TeamThreadRange(member, amcd.nParticles), + Kokkos::ThreadVectorRange(member, amcd.nParticles), [&](const ordinal_type& i_part) { for (size_t i_simpol = 0; i_simpol < amcd.nSimpol_tran; i_simpol++) { From 71e2b003e5d00a57c83c0fd9a21ea7314cc3c88c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 17 Sep 2024 09:40:51 -0600 Subject: [PATCH 12/19] Writing outputs only if verbose is true. --- src/examples/TChem_AerosolChemistry.cpp | 99 ++++++++++++------------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/src/examples/TChem_AerosolChemistry.cpp b/src/examples/TChem_AerosolChemistry.cpp index 55586952..3b8a6f6f 100644 --- a/src/examples/TChem_AerosolChemistry.cpp +++ b/src/examples/TChem_AerosolChemistry.cpp @@ -128,11 +128,11 @@ int main(int argc, char *argv[]) { input_file_particles=chemFile; } -#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) +#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) printf(" TChem is running ATM model box with KokkosKernels\n"); -#else +#else printf(" TChem is running ATM model box with Tines\n"); -#endif +#endif TChem::exec_space().print_configuration(std::cout, detail); TChem::host_exec_space().print_configuration(std::cout, detail); @@ -160,7 +160,10 @@ int main(int argc, char *argv[]) { const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); - FILE *fout = fopen(outputFile.c_str(), "w"); + FILE *fout; + if (verbose) { + fout = fopen(outputFile.c_str(), "w"); + } // read scenario condition from yaml file // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; @@ -173,10 +176,10 @@ int main(int argc, char *argv[]) { real_type_2d_view_host num_concentration_host; real_type_2d_view_host state_host; - + // scenario particles amd.scenarioConditionParticles(input_file_particles, nbacth_files, num_concentration_host, state_scenario_host); - + real_type_2d_view num_concentration; real_type_2d_view state; @@ -191,7 +194,7 @@ int main(int argc, char *argv[]) { auto state_at_0 = Kokkos::subview(state, 0, Kokkos::ALL); Kokkos::deep_copy(state_at_0, state_scenario_host_at_0); TChem::Test::cloneView(state); - //make sure to copy state to host view. + //make sure to copy state to host view. Kokkos::deep_copy(state_host, state); //scenario particles auto num_concentration_host_at_0 = Kokkos::subview(num_concentration_host, 0, Kokkos::ALL); @@ -206,7 +209,7 @@ int main(int argc, char *argv[]) { Kokkos::deep_copy(state, state_scenario_host); state_host = state_scenario_host; - // scenario particles + // scenario particles num_concentration = real_type_2d_view("num_concentration", nBatch, amd.nParticles_); Kokkos::deep_copy(num_concentration, num_concentration_host); } @@ -238,14 +241,14 @@ int main(int argc, char *argv[]) { using policy_type = typename TChem::UseThisTeamPolicy::type; -#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) +#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) /// team policy // The Kokkos-kernels BDF solver is designed for range policy. Therefore, I will use a team size of 1. policy_type policy(exec_space_instance, nBatch, 1); -#else +#else policy_type policy(exec_space_instance, nBatch, Kokkos::AUTO()); -#endif +#endif ordinal_type number_of_equations(0); @@ -258,11 +261,11 @@ int main(int argc, char *argv[]) { const ordinal_type level = 1; ordinal_type per_team_extent(0); -#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) +#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) per_team_extent = TChem::AerosolChemistry_KokkosKernels::getWorkSpaceSize(kmcd, amcd); #else per_team_extent = TChem::AerosolChemistry::getWorkSpaceSize(kmcd, amcd); -#endif +#endif const ordinal_type per_team_scratch = TChem::Scratch::shmem_size(per_team_extent); policy.set_scratch_size(level, Kokkos::PerTeam(per_team_scratch)); @@ -314,35 +317,33 @@ int main(int argc, char *argv[]) { dt_host = real_type_1d_view_host("dt host", nBatch); ordinal_type iter = 0; - - fprintf(fout, "%s \t %s \t %s \t ", "iter", "t", "dt"); - fprintf(fout, "%s \t %s \t %s \t", "Density[kg/m3]", "Pressure[Pascal]", + if (verbose) { + fprintf(fout, "%s \t %s \t %s \t ", "iter", "t", "dt"); + fprintf(fout, "%s \t %s \t %s \t", "Density[kg/m3]", "Pressure[Pascal]", "Temperature[K]"); - for (ordinal_type k = 0; k < kmcd.nSpec; k++) - fprintf(fout, "%s \t", &speciesNamesHost(k, 0)); - //given aero idx return species name - std::map aero_idx_sp_name; - for (std::map::iterator - i = amd.aerosol_sp_name_idx_.begin(); - i != amd.aerosol_sp_name_idx_.end(); ++i) - aero_idx_sp_name[i->second] = i->first; + for (ordinal_type k = 0; k < kmcd.nSpec; k++) + fprintf(fout, "%s \t", &speciesNamesHost(k, 0)); + //given aero idx return species name + std::map aero_idx_sp_name; + for (std::map::iterator + i = amd.aerosol_sp_name_idx_.begin(); + i != amd.aerosol_sp_name_idx_.end(); ++i) + aero_idx_sp_name[i->second] = i->first; - for (ordinal_type ipart = 0; ipart < amd.nParticles_; ipart++) - { - for (ordinal_type isp = 0; isp < amd.nSpec_; isp++) + for (ordinal_type ipart = 0; ipart < amd.nParticles_; ipart++) { - // std::cout << "species Name : "<< aero_idx_sp_name[i] << "\n"; - auto aero_sp_name = aero_idx_sp_name[isp]+"_p"+std::to_string(ipart); - fprintf(fout, "%s \t", aero_sp_name.c_str()); - }// isp - }// ipar - - - - fprintf(fout, "\n"); - writeState(-1, t_host, dt_host, state_host, fout); - + for (ordinal_type isp = 0; isp < amd.nSpec_; isp++) + { + // std::cout << "species Name : "<< aero_idx_sp_name[i] << "\n"; + auto aero_sp_name = aero_idx_sp_name[isp]+"_p"+std::to_string(ipart); + fprintf(fout, "%s \t", aero_sp_name.c_str()); + }// isp + }// ipar + + fprintf(fout, "\n"); + writeState(-1, t_host, dt_host, state_host, fout); + } real_type tsum(0); Kokkos::Timer timer; @@ -351,7 +352,7 @@ int main(int argc, char *argv[]) { timer.reset(); -#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) +#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) TChem::AerosolChemistry_KokkosKernels::runDeviceBatch( policy, tol_time, fac, tadv, state, num_concentration, t, dt, state, kmcd, amcd); @@ -367,15 +368,12 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "\"%s%d\": %20.14e, \n", "wall_time_iter_", iter, t_device_batch); - - - Kokkos::deep_copy(dt_host, dt); - Kokkos::deep_copy(t_host, t); - Kokkos::deep_copy(state_host, state); - - - writeState(iter, t_host, dt_host, state_host, fout); - + if (verbose) { + Kokkos::deep_copy(dt_host, dt); + Kokkos::deep_copy(t_host, t); + Kokkos::deep_copy(state_host, state); + writeState(iter, t_host, dt_host, state_host, fout); + } /// carry over time and dt computed in this step tsum = zero; Kokkos::parallel_reduce( @@ -400,8 +398,9 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "%s: %d \n", "\"number_of_samples\"", nBatch); fprintf(fout_times, "} \n "); // reaction rates - - fclose(fout); + if (verbose) { + fclose(fout); + } fprintf(fout_times, "}\n "); // end index time fclose(fout_times); } From 3b1fd035e33b2087ad1e9918a8cdbd6e8fdb84f6 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 17 Sep 2024 09:55:29 -0600 Subject: [PATCH 13/19] Turn off writing output files for numerical experiments. --- src/examples/TChem_AerosolChemistry.cpp | 14 +++- src/examples/TChem_AerosolChemistry_CVODE.cpp | 73 +++++++++++-------- 2 files changed, 53 insertions(+), 34 deletions(-) diff --git a/src/examples/TChem_AerosolChemistry.cpp b/src/examples/TChem_AerosolChemistry.cpp index 3b8a6f6f..370bacf5 100644 --- a/src/examples/TChem_AerosolChemistry.cpp +++ b/src/examples/TChem_AerosolChemistry.cpp @@ -160,8 +160,16 @@ int main(int argc, char *argv[]) { const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); + + bool write_time_profiles(true); + // For scaling studies, we must execute this example many times. + // Thus, we do not want to write the solution to a file. + // In those cases, we pass outputFile=None. + if (outputFile =="None"){ + write_time_profiles=false; + }// FILE *fout; - if (verbose) { + if (write_time_profiles) { fout = fopen(outputFile.c_str(), "w"); } // read scenario condition from yaml file @@ -317,7 +325,7 @@ int main(int argc, char *argv[]) { dt_host = real_type_1d_view_host("dt host", nBatch); ordinal_type iter = 0; - if (verbose) { + if (write_time_profiles) { fprintf(fout, "%s \t %s \t %s \t ", "iter", "t", "dt"); fprintf(fout, "%s \t %s \t %s \t", "Density[kg/m3]", "Pressure[Pascal]", "Temperature[K]"); @@ -398,7 +406,7 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "%s: %d \n", "\"number_of_samples\"", nBatch); fprintf(fout_times, "} \n "); // reaction rates - if (verbose) { + if (write_time_profiles) { fclose(fout); } fprintf(fout_times, "}\n "); // end index time diff --git a/src/examples/TChem_AerosolChemistry_CVODE.cpp b/src/examples/TChem_AerosolChemistry_CVODE.cpp index 2da26e72..33894649 100644 --- a/src/examples/TChem_AerosolChemistry_CVODE.cpp +++ b/src/examples/TChem_AerosolChemistry_CVODE.cpp @@ -163,9 +163,19 @@ int main(int argc, char *argv[]) { const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); - FILE *fout = fopen(outputFile.c_str(), "w"); + bool write_time_profiles(true); + // For scaling studies, we must execute this example many times. + // Thus, we do not want to write the solution to a file. + // In those cases, we pass outputFile=None. + if (outputFile =="None"){ + write_time_profiles=false; + }// + FILE *fout; + if (write_time_profiles) { + fout = fopen(outputFile.c_str(), "w"); + } + // read scenario condition from yaml file // read scenario condition from yaml file - // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; ordinal_type nbacth_files=1; printf("conditions parsing ...\n"); @@ -200,7 +210,7 @@ int main(int argc, char *argv[]) { nBatch = nbacth_files; state = state_scenario_host; - // scenario particles + // scenario particles num_concentration = real_type_2d_view_host("num_concentration", nBatch, amd.nParticles_); Kokkos::deep_copy(num_concentration, num_concentration_scenario); } @@ -310,33 +320,32 @@ int main(int argc, char *argv[]) { ordinal_type iter = 0; - fprintf(fout, "%s \t %s \t %s \t ", "iter", "t", "dt"); - fprintf(fout, "%s \t %s \t %s \t", "Density[kg/m3]", "Pressure[Pascal]", + if (write_time_profiles) { + fprintf(fout, "%s \t %s \t %s \t ", "iter", "t", "dt"); + fprintf(fout, "%s \t %s \t %s \t", "Density[kg/m3]", "Pressure[Pascal]", "Temperature[K]"); - for (ordinal_type k = 0; k < kmcd.nSpec; k++) - fprintf(fout, "%s \t", &speciesNamesHost(k, 0)); - //given aero idx return species name - std::map aero_idx_sp_name; - for (std::map::iterator - i = amd.aerosol_sp_name_idx_.begin(); - i != amd.aerosol_sp_name_idx_.end(); ++i) - aero_idx_sp_name[i->second] = i->first; - - for (ordinal_type ipart = 0; ipart < amd.nParticles_; ipart++) - { - for (ordinal_type isp = 0; isp < amd.nSpec_; isp++) + for (ordinal_type k = 0; k < kmcd.nSpec; k++) + fprintf(fout, "%s \t", &speciesNamesHost(k, 0)); + //given aero idx return species name + std::map aero_idx_sp_name; + for (std::map::iterator + i = amd.aerosol_sp_name_idx_.begin(); + i != amd.aerosol_sp_name_idx_.end(); ++i) + aero_idx_sp_name[i->second] = i->first; + for (ordinal_type ipart = 0; ipart < amd.nParticles_; ipart++) { - // std::cout << "species Name : "<< aero_idx_sp_name[i] << "\n"; - auto aero_sp_name = aero_idx_sp_name[isp]+"_p"+std::to_string(ipart); - fprintf(fout, "%s \t", aero_sp_name.c_str()); - }// isp - }// ipar - - - - fprintf(fout, "\n"); - writeState(-1, t, dt, state, fout); + for (ordinal_type isp = 0; isp < amd.nSpec_; isp++) + { + // std::cout << "species Name : "<< aero_idx_sp_name[i] << "\n"; + auto aero_sp_name = aero_idx_sp_name[isp]+"_p"+std::to_string(ipart); + fprintf(fout, "%s \t", aero_sp_name.c_str()); + }// isp + }// ipar + + fprintf(fout, "\n"); + writeState(-1, t, dt, state, fout); + }// write_time_profiles real_type tsum(0); Kokkos::Timer timer; @@ -355,8 +364,9 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "\"%s%d\": %20.14e, \n", "wall_time_iter_", iter, t_device_batch); - writeState(iter, t, dt, state, fout); - + if (write_time_profiles) { + writeState(iter, t, dt, state, fout); + } /// carry over time and dt computed in this step tsum = zero; Kokkos::parallel_reduce( @@ -381,8 +391,9 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "%s: %d \n", "\"number_of_samples\"", nBatch); fprintf(fout_times, "} \n "); // reaction rates - - fclose(fout); + if (write_time_profiles) { + fclose(fout); + } fprintf(fout_times, "}\n "); // end index time fclose(fout_times); } From 91deed02259b4f87326239457368e2ce29bb4772 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 17 Sep 2024 11:57:08 -0600 Subject: [PATCH 14/19] write-time-profiles as command input --- src/examples/TChem_AerosolChemistry.cpp | 14 +++++------ src/examples/TChem_AerosolChemistry_CVODE.cpp | 25 ++++++++----------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/examples/TChem_AerosolChemistry.cpp b/src/examples/TChem_AerosolChemistry.cpp index 370bacf5..7bb90b8c 100644 --- a/src/examples/TChem_AerosolChemistry.cpp +++ b/src/examples/TChem_AerosolChemistry.cpp @@ -61,6 +61,10 @@ int main(int argc, char *argv[]) { std::string inputFile("None"); std::string input_file_particles("None"); bool use_cloned_samples(false); + // For scaling studies, we must execute this example many times. + // Thus, we do not want to write the solution to a file. + // In those cases, we pass write_time_profiles false. + bool write_time_profiles(true); /// parse command line arguments TChem::CommandLineParser opts( "This example computes the solution of gas chemistry problem"); @@ -103,6 +107,8 @@ int main(int argc, char *argv[]) { &max_num_newton_iterations); opts.set_option( "verbose", "If true, printout the first Jacobian values", &verbose); + opts.set_option( + "write-time-profiles", "If true, this example will write the time profile output to a file.", &write_time_profiles); opts.set_option("team_thread_size", "time thread size ", &team_size); opts.set_option("batch_size", " number of batches or samples, e.g. 10 ", &nBatch); opts.set_option("vector_thread_size", "vector thread size ", @@ -160,14 +166,6 @@ int main(int argc, char *argv[]) { const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); - - bool write_time_profiles(true); - // For scaling studies, we must execute this example many times. - // Thus, we do not want to write the solution to a file. - // In those cases, we pass outputFile=None. - if (outputFile =="None"){ - write_time_profiles=false; - }// FILE *fout; if (write_time_profiles) { fout = fopen(outputFile.c_str(), "w"); diff --git a/src/examples/TChem_AerosolChemistry_CVODE.cpp b/src/examples/TChem_AerosolChemistry_CVODE.cpp index 33894649..276d18f8 100644 --- a/src/examples/TChem_AerosolChemistry_CVODE.cpp +++ b/src/examples/TChem_AerosolChemistry_CVODE.cpp @@ -62,7 +62,10 @@ int main(int argc, char *argv[]) { std::string inputFile("None"); std::string input_file_particles("None"); bool use_cloned_samples(false); - bool use_cvode(false); + // For scaling studies, we must execute this example many times. + // Thus, we do not want to write the solution to a file. + // In those cases, we pass write_time_profiles false. + bool write_time_profiles(true); /// parse command line arguments TChem::CommandLineParser opts( "This example computes the solution of gas chemistry problem"); @@ -105,6 +108,7 @@ int main(int argc, char *argv[]) { &max_num_newton_iterations); opts.set_option( "verbose", "If true, printout the first Jacobian values", &verbose); + opts.set_option("team_thread_size", "time thread size ", &team_size); opts.set_option("batch_size", " number of batches or samples, e.g. 10 ", &nBatch); opts.set_option("vector_thread_size", "vector thread size ", @@ -112,9 +116,8 @@ int main(int argc, char *argv[]) { opts.set_option("atol-time", "Absolute tolerance used for adaptive time stepping", &atol_time); opts.set_option( "use_cloned_samples", "If true, one state vector will be cloned.", &use_cloned_samples); - opts.set_option("use-cvode", - "if true code runs ignition zero d reactor with cvode; otherwise, it uses TrBDF2", - &use_cvode); + opts.set_option( + "write-time-profiles", "If true, this example will write the time profile output to a file.", &write_time_profiles) const bool r_parse = opts.parse(argc, argv); if (r_parse) @@ -163,13 +166,6 @@ int main(int argc, char *argv[]) { const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); - bool write_time_profiles(true); - // For scaling studies, we must execute this example many times. - // Thus, we do not want to write the solution to a file. - // In those cases, we pass outputFile=None. - if (outputFile =="None"){ - write_time_profiles=false; - }// FILE *fout; if (write_time_profiles) { fout = fopen(outputFile.c_str(), "w"); @@ -258,11 +254,10 @@ int main(int argc, char *argv[]) { using time_integrator_cvode_type = Tines::TimeIntegratorCVODE; Tines::value_type_1d_view cvodes; - if (use_cvode) { - cvodes = Tines::value_type_1d_view("cvodes", nBatch); - for (ordinal_type i=0;i("cvodes", nBatch); + for (ordinal_type i=0;i Date: Tue, 17 Sep 2024 12:02:24 -0600 Subject: [PATCH 15/19] Fixing compilation error. --- src/examples/TChem_AerosolChemistry_CVODE.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/examples/TChem_AerosolChemistry_CVODE.cpp b/src/examples/TChem_AerosolChemistry_CVODE.cpp index 276d18f8..cb3dc48d 100644 --- a/src/examples/TChem_AerosolChemistry_CVODE.cpp +++ b/src/examples/TChem_AerosolChemistry_CVODE.cpp @@ -117,7 +117,7 @@ int main(int argc, char *argv[]) { opts.set_option( "use_cloned_samples", "If true, one state vector will be cloned.", &use_cloned_samples); opts.set_option( - "write-time-profiles", "If true, this example will write the time profile output to a file.", &write_time_profiles) + "write-time-profiles", "If true, this example will write the time profile output to a file.", &write_time_profiles); const bool r_parse = opts.parse(argc, argv); if (r_parse) From 28d29bb85af70d3201ab2d7ccb030c1af1295574 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 16 Oct 2024 08:53:11 -0600 Subject: [PATCH 16/19] Fixing typo. --- src/examples/TChem_AerosolChemistry_RHSs.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/examples/TChem_AerosolChemistry_RHSs.cpp b/src/examples/TChem_AerosolChemistry_RHSs.cpp index ae773700..010a6ae2 100644 --- a/src/examples/TChem_AerosolChemistry_RHSs.cpp +++ b/src/examples/TChem_AerosolChemistry_RHSs.cpp @@ -122,10 +122,10 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file real_type_2d_view_host_type state_scenario_host; - ordinal_type nbacth_files=1; + ordinal_type nbatch_files=1; printf("conditions parsing ...\n"); TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); printf("Number of const species %d \n", kmcd.nConstSpec); real_type_2d_view_host_type num_concentration_host; @@ -133,15 +133,15 @@ int main(int argc, char *argv[]) { // scenario particles amd.scenarioConditionParticles(input_file_particles, - nbacth_files, + nbatch_files, num_concentration_host, state_scenario_host); real_type_2d_view_type num_concentration; real_type_2d_view_type state; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -161,7 +161,7 @@ int main(int argc, char *argv[]) { TChem::Test::cloneView(num_concentration); } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view_type("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); state_host = state_scenario_host; From ff899f440ba9fc4ccd02b452324cbf79d2ef098b Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 16 Oct 2024 08:54:56 -0600 Subject: [PATCH 17/19] Fixing typo. --- src/examples/TChem_AerosolChemistry.cpp | 12 ++++++------ src/examples/TChem_AerosolChemistry_CVODE.cpp | 12 ++++++------ src/examples/TChem_AtmosphericChemistryE3SM.cpp | 14 +++++++------- .../TChem_AtmosphericChemistryE3SM_CVODE.cpp | 14 +++++++------- .../TChem_Linozv3_StratosphereChemSolver.cpp | 2 +- src/examples/TChem_NetProductionRates.cpp | 16 ++++++++-------- src/examples/TChem_RateofProgress.cpp | 14 +++++++------- src/examples/TChem_ReactionRates.cpp | 10 +++++----- 8 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/examples/TChem_AerosolChemistry.cpp b/src/examples/TChem_AerosolChemistry.cpp index 7bb90b8c..4a222d10 100644 --- a/src/examples/TChem_AerosolChemistry.cpp +++ b/src/examples/TChem_AerosolChemistry.cpp @@ -173,10 +173,10 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=1; + ordinal_type nbatch_files=1; printf("conditions parsing ...\n"); TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); printf("Number of const species %d \n", kmcd.nConstSpec); real_type_2d_view_host num_concentration_host; @@ -184,13 +184,13 @@ int main(int argc, char *argv[]) { // scenario particles - amd.scenarioConditionParticles(input_file_particles, nbacth_files, num_concentration_host, state_scenario_host); + amd.scenarioConditionParticles(input_file_particles, nbatch_files, num_concentration_host, state_scenario_host); real_type_2d_view num_concentration; real_type_2d_view state; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -210,7 +210,7 @@ int main(int argc, char *argv[]) { TChem::Test::cloneView(num_concentration); } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); state_host = state_scenario_host; diff --git a/src/examples/TChem_AerosolChemistry_CVODE.cpp b/src/examples/TChem_AerosolChemistry_CVODE.cpp index cb3dc48d..33d80e74 100644 --- a/src/examples/TChem_AerosolChemistry_CVODE.cpp +++ b/src/examples/TChem_AerosolChemistry_CVODE.cpp @@ -173,19 +173,19 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=1; + ordinal_type nbatch_files=1; printf("conditions parsing ...\n"); TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); const ordinal_type n_active_vars = total_n_species - kmcd.nConstSpec; printf("Number of const species %d \n", kmcd.nConstSpec); real_type_2d_view_host num_concentration_scenario, num_concentration; - amd.scenarioConditionParticles(input_file_particles, nbacth_files, num_concentration_scenario, state_scenario_host); + amd.scenarioConditionParticles(input_file_particles, nbatch_files, num_concentration_scenario, state_scenario_host); real_type_2d_view_host state; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -203,7 +203,7 @@ int main(int argc, char *argv[]) { TChem::Test::cloneView(num_concentration); } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = state_scenario_host; // scenario particles diff --git a/src/examples/TChem_AtmosphericChemistryE3SM.cpp b/src/examples/TChem_AtmosphericChemistryE3SM.cpp index 76b8e161..0b385204 100644 --- a/src/examples/TChem_AtmosphericChemistryE3SM.cpp +++ b/src/examples/TChem_AtmosphericChemistryE3SM.cpp @@ -153,16 +153,16 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec,stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec,stateVecDim, state_scenario_host, nbatch_files); // read photolysis reaction values // we assume photolysis reaction are computed by another tool. real_type_2d_view_host photo_rates_scenario_host; ordinal_type n_photo_rates = 0; TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(inputFile, - nbacth_files, + nbatch_files, // output photo_rates_scenario_host, n_photo_rates @@ -174,7 +174,7 @@ int main(int argc, char *argv[]) { printf("Number of species %d \n", kmcd.nSpec); printf("Number of const species %d \n", kmcd.nConstSpec); // Note: We allocate external_sources_scenario_host - real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbacth_files,n_active_vars); + real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbatch_files,n_active_vars); TChem::AtmChemistry::setScenarioConditionsExternalForcing(inputFile, speciesNamesHost, // output @@ -186,8 +186,8 @@ int main(int argc, char *argv[]) { real_type_2d_view_host state_host; real_type_2d_view external_sources; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -224,7 +224,7 @@ int main(int argc, char *argv[]) { } } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); state_host = state_scenario_host; diff --git a/src/examples/TChem_AtmosphericChemistryE3SM_CVODE.cpp b/src/examples/TChem_AtmosphericChemistryE3SM_CVODE.cpp index 12cb98c7..a2575b82 100644 --- a/src/examples/TChem_AtmosphericChemistryE3SM_CVODE.cpp +++ b/src/examples/TChem_AtmosphericChemistryE3SM_CVODE.cpp @@ -152,16 +152,16 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); // read photolysis reaction values // we assume photolysis reaction are computed by another tool. real_type_2d_view_host photo_rates_scenario_host; ordinal_type n_photo_rates = 0; TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(inputFile, - nbacth_files, + nbatch_files, // output photo_rates_scenario_host, n_photo_rates @@ -173,7 +173,7 @@ int main(int argc, char *argv[]) { printf("Number of species %d \n", kmcd.nSpec); printf("Number of const species %d \n", kmcd.nConstSpec); // Note: We allocate external_sources_scenario_host - real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbacth_files,n_active_vars); + real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbatch_files,n_active_vars); TChem::AtmChemistry::setScenarioConditionsExternalForcing(inputFile, speciesNamesHost, // output @@ -184,8 +184,8 @@ int main(int argc, char *argv[]) { real_type_2d_view_host photo_rates; real_type_2d_view_host external_sources; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -216,7 +216,7 @@ int main(int argc, char *argv[]) { } } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = state_scenario_host; if (n_photo_rates > 0 ) diff --git a/src/examples/TChem_Linozv3_StratosphereChemSolver.cpp b/src/examples/TChem_Linozv3_StratosphereChemSolver.cpp index 6571de82..68e2cca1 100644 --- a/src/examples/TChem_Linozv3_StratosphereChemSolver.cpp +++ b/src/examples/TChem_Linozv3_StratosphereChemSolver.cpp @@ -373,7 +373,7 @@ void linv3_stratosphere_solver_test( const int team_size, if (team_size > 0 && vector_size > 0) { policy = policy_type(exec_space_instance, nBatch, team_size, vector_size); } else if (team_size > 0 && vector_size < 0) { - // only set team size + // only set team size policy = policy_type(exec_space_instance, nBatch, team_size); } diff --git a/src/examples/TChem_NetProductionRates.cpp b/src/examples/TChem_NetProductionRates.cpp index f2517cd1..737b1376 100644 --- a/src/examples/TChem_NetProductionRates.cpp +++ b/src/examples/TChem_NetProductionRates.cpp @@ -82,9 +82,9 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); // read photolysis reaction values @@ -92,7 +92,7 @@ int main(int argc, char *argv[]) { real_type_2d_view_host photo_rates_scenario_host; ordinal_type n_photo_rates = 0; TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(inputFile, - nbacth_files, + nbatch_files, // output photo_rates_scenario_host, n_photo_rates @@ -105,7 +105,7 @@ int main(int argc, char *argv[]) { printf("Number of species %d \n", kmcd.nSpec); printf("Number of const species %d \n", kmcd.nConstSpec); // Note: We allocate external_sources_scenario_host - real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbacth_files,n_active_vars); + real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbatch_files,n_active_vars); TChem::AtmChemistry::setScenarioConditionsExternalForcing(inputFile, speciesNamesHost, // output @@ -116,8 +116,8 @@ int main(int argc, char *argv[]) { real_type_2d_view photo_rates; real_type_2d_view external_sources; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -155,13 +155,13 @@ int main(int argc, char *argv[]) { } } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); if (n_photo_rates > 0 ) { - nBatch = nbacth_files; + nBatch = nbatch_files; photo_rates = real_type_2d_view("StateVector Devices", nBatch, n_photo_rates); Kokkos::deep_copy(photo_rates, photo_rates_scenario_host); diff --git a/src/examples/TChem_RateofProgress.cpp b/src/examples/TChem_RateofProgress.cpp index 70ed9975..7c1210ce 100644 --- a/src/examples/TChem_RateofProgress.cpp +++ b/src/examples/TChem_RateofProgress.cpp @@ -84,16 +84,16 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); // read photolysis reaction values // we assume photolysis reaction are computed by another tool. real_type_2d_view_host photo_rates_scenario_host; ordinal_type n_photo_rates = 0; TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(inputFile, - nbacth_files, + nbatch_files, // output photo_rates_scenario_host, n_photo_rates @@ -102,8 +102,8 @@ int main(int argc, char *argv[]) { real_type_2d_view state; real_type_2d_view photo_rates; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -127,13 +127,13 @@ int main(int argc, char *argv[]) { } // n_photo_rates } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); if (n_photo_rates > 0 ) { - nBatch = nbacth_files; + nBatch = nbatch_files; photo_rates = real_type_2d_view("StateVector Devices", nBatch, n_photo_rates); Kokkos::deep_copy(photo_rates, photo_rates_scenario_host); diff --git a/src/examples/TChem_ReactionRates.cpp b/src/examples/TChem_ReactionRates.cpp index 8e9d03db..65808537 100644 --- a/src/examples/TChem_ReactionRates.cpp +++ b/src/examples/TChem_ReactionRates.cpp @@ -78,14 +78,14 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(chemFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); real_type_2d_view state; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) { TChem::Test::cloneView(state_host); Kokkos::deep_copy(state, state_host); } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); } From c8b26c3dcac7a99559941f1b614675e0ead04469 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 16 Oct 2024 08:57:37 -0600 Subject: [PATCH 18/19] Fixing typo. --- src/examples/TChem_AerosolChemistry.cpp | 2 +- src/examples/TChem_AerosolChemistry_CVODE.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/examples/TChem_AerosolChemistry.cpp b/src/examples/TChem_AerosolChemistry.cpp index 4a222d10..b984904a 100644 --- a/src/examples/TChem_AerosolChemistry.cpp +++ b/src/examples/TChem_AerosolChemistry.cpp @@ -219,7 +219,7 @@ int main(int argc, char *argv[]) { num_concentration = real_type_2d_view("num_concentration", nBatch, amd.nParticles_); Kokkos::deep_copy(num_concentration, num_concentration_host); } - printf("Number of nbacth %d \n",nBatch); + printf("Number of nbatch %d \n",nBatch); auto writeState = [](const ordinal_type iter, const real_type_1d_view_host _t, const real_type_1d_view_host _dt, diff --git a/src/examples/TChem_AerosolChemistry_CVODE.cpp b/src/examples/TChem_AerosolChemistry_CVODE.cpp index 33d80e74..16246218 100644 --- a/src/examples/TChem_AerosolChemistry_CVODE.cpp +++ b/src/examples/TChem_AerosolChemistry_CVODE.cpp @@ -210,7 +210,7 @@ int main(int argc, char *argv[]) { num_concentration = real_type_2d_view_host("num_concentration", nBatch, amd.nParticles_); Kokkos::deep_copy(num_concentration, num_concentration_scenario); } - printf("Number of nbacth %d \n",nBatch); + printf("Number of nbatch %d \n",nBatch); auto writeState = [](const ordinal_type iter, const real_type_1d_view_host _t, const real_type_1d_view_host _dt, From a46944857463a908fef46e25523c5c135777b434 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 16 Oct 2024 08:59:45 -0600 Subject: [PATCH 19/19] Fixing typo. --- src/core/TChem_AtmosphericChemistry.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/TChem_AtmosphericChemistry.cpp b/src/core/TChem_AtmosphericChemistry.cpp index f4eb1fbb..9a1076ae 100644 --- a/src/core/TChem_AtmosphericChemistry.cpp +++ b/src/core/TChem_AtmosphericChemistry.cpp @@ -539,9 +539,9 @@ TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(const std::string& if (nBatch_photo_reation == nBatch) { // std::cout << photo_rate_values << "\n"; - for (int ibacth = 0; ibacth < nBatch; ++ibacth) + for (int ibatch = 0; ibatch < nBatch; ++ibatch) { - photo_rates_host(ibacth, ireac) = photo_rate_values[ibacth].as(); + photo_rates_host(ibatch, ireac) = photo_rate_values[ibatch].as(); } } else { printf("Error number of values in photo reaction is different than number of conditions"); @@ -610,9 +610,9 @@ if (root["external_forcing"]){ if (nBatch_ext_forcing == nBatch) { - for (int ibacth = 0; ibacth < nBatch; ++ibacth) + for (int ibatch = 0; ibatch < nBatch; ++ibatch) { - external_forcing_host(ibacth, sp_idx) = ext_forcing_values[ibacth].as(); + external_forcing_host(ibatch, sp_idx) = ext_forcing_values[ibatch].as(); } } else { printf("Error number of values in external forcing is different than number of conditions");