diff --git a/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh new file mode 100755 index 00000000..4c2ae18c --- /dev/null +++ b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/inputs.sh @@ -0,0 +1,8 @@ +inputs=${TCHEM_INSTALL_PATH}/examples/runs/atmospheric_chemistry/CB05CL_AE5_w_simpolSOA +inputfile_particles=${inputs}/scenario_conditions_particle.yaml +chemfile=${inputs}/config_full_gas.yaml +aerofile=${inputs}/mechanism_aero.yaml + +export scenario_n_inputs="--chemfile=$chemfile \ + --aerofile=$aerofile \ + --inputfile_particles=$inputfile_particles" diff --git a/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runCUDA.sh b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runCUDA.sh new file mode 100755 index 00000000..c4df4714 --- /dev/null +++ b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runCUDA.sh @@ -0,0 +1,23 @@ +export DEVICE=GPU +export experiment_exe_root=${experiment_base}${experiment_suffix} +export sacado_flag=$sflag +echo "sacado flag = ${sacado_flag}" +source ../loadGPU_WS.sh +source ./inputs.sh + +export experiment_name="${experiment_name}-${sacado_flag}" +echo "experiment name is ${experiment_name}" +echo "experiment exe is ${exec}" +use_cloned_samples=true +# change to true if you want to output the reaction rates outputs. +verbose=false +#we will save outputs in this directory +tchem_outputs=CUDA +mkdir -p ${tchem_outputs}/${experiment_name} +nbatch=(72 144 216) +# 288 360 432 504 576 648 720 792 864 936 1008) +# let's use for now 1 and 1 for team and vector size; however, we need to tune up these parameters. +vector_size=(1) +team_size=(1) +nparticles=(1 10 100 1000) +source ../runThisGPU.sh diff --git a/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runHOSTWS.sh b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runHOSTWS.sh new file mode 100755 index 00000000..17dd665a --- /dev/null +++ b/scripts/aerosol_performance/CB05CL_AE5_w_simpolSOA/runHOSTWS.sh @@ -0,0 +1,24 @@ +export DEVICE=CPU +export experiment_exe_root=${experiment_base}${experiment_suffix} +export sacado_flag=$sflag +echo "sacado flag = ${sacado_flag}" +source ../loadCPU_WS.sh +source ./inputs.sh + +export experiment_name="${experiment_name}-${sacado_flag}" +echo "experiment name is ${experiment_name}" +echo "experiment exe is ${exec}" +use_cloned_samples=true +# change to true if you want to output the reaction rates outputs. +verbose=false +#we will save outputs in this directory +tchem_outputs=HOST +mkdir -p ${tchem_outputs}/${experiment_name} +#20 100 1000 10000 100000 200000 500000 1000000 2000000 3000000 4500000 +nbatch=(1) +# 144 216 288) +# 360 432 504 576 648 720 792 864 936 1008) +Nthread=(52 104) +nparticles=(1 10 50 100) +# 500 1000 5000 10000 +source ../runThisCPU.sh diff --git a/scripts/aerosol_performance/loadMachinePathsTemplate.sh b/scripts/aerosol_performance/loadMachinePathsTemplate.sh new file mode 100644 index 00000000..f009fca5 --- /dev/null +++ b/scripts/aerosol_performance/loadMachinePathsTemplate.sh @@ -0,0 +1,6 @@ + +export KOKKOS_TOOLS_LIBS=/path/kokkostools/libs/libkp_kernel_timer.so +TCHEM_INSTALL_PATH=/path/tchem/install +exec=${TCHEM_INSTALL_PATH}/examples/${experiment_exe_root}.x +machine_name=/name/machine + diff --git a/scripts/aerosol_performance/runAll.sh b/scripts/aerosol_performance/runAll.sh new file mode 100755 index 00000000..269ca8bf --- /dev/null +++ b/scripts/aerosol_performance/runAll.sh @@ -0,0 +1,125 @@ +#measure wall times for the following mechanisms +dirs=(CB05CL_AE5_w_simpolSOA) +sacado_flags=(no_sacado) +export experiment_base=TChem_AerosolChemistry + +cpu_solver_strList=( + # "trbdf" + # "kokkoskernels" + # "cvode" + "rhss" +) +cpu_exe_string=( + # "" + # "_KokkosKernels" + # "_CVODE" + "_RHSs" +) +gpu_solver_strList=( + # "trbdf" + # "kokkoskernels" + #"rhss" +) +gpu_exe_string=( + # "" + # "_KokkosKernels" + #"_RHSs" +) + +rhss_params(){ + export numerical_params="" +} +trbdf_params(){ + t_iterPerInt=1 + min_dt='1e-3' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --atol-time=${atol_t} \ + --tend=$tend \ + --atol-newton=1e-18 \ + --rtol-newton=1e-8 \ + --max-newton-iterations=20 \ + --max-time-iterations=20000 " + +} + +kokkoskernels_params(){ + t_iterPerInt=1 + min_dt='1e-1' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --atol-time=${atol_t} \ + --tend=$tend " +} + +cvode_params(){ + t_iterPerInt=10 + min_dt='1e-20' + max_dt='1' + atol_t='1e-12' + tol_time='1e-3' + tend='1' + export numerical_params="--tol-time=$tol_time \ + --time-iterations-per-interval=$t_iterPerInt \ + --dtmin=$min_dt \ + --dtmax=$max_dt \ + --use-cvode=true \ + --atol-time=${atol_t} \ + --tend=$tend " +} + +exe=runHOSTWS.sh +for i in "${!cpu_solver_strList[@]}"; do + export experiment_name="${cpu_solver_strList[i]}" + export experiment_suffix="${cpu_exe_string[i]}" + params_fxn="${experiment_name}_params" + printf "setting variables for %s case (CPU)\n" "${experiment_name}" + ${params_fxn} + for dir in ${dirs[@]}; do + for sacado in ${sacado_flags[@]}; do + if [ "${experiment_name}" == "expEuler" ] && [ "${sacado}" == "sacado" ]; then + continue + elif [ "${experiment_name}" == "cvode" ] && [ "${sacado}" == "sacado" ]; then + continue + fi + export sflag=$sacado + run_this="cd $dir;./$exe;cd -" + echo $run_this + eval $run_this + done + done + echo "end of outer loop" +done + +exe=runCUDA.sh +for i in "${!gpu_solver_strList[@]}"; do + export experiment_name="${gpu_solver_strList[i]}" + export experiment_suffix="${gpu_exe_string[i]}" + params_fxn="${experiment_name}_params" + printf "setting variables for %s case (GPU)\n" "${experiment_name}" + ${params_fxn} + for dir in ${dirs[@]}; do + for sacado in ${sacado_flags[@]}; do + if [ "${experiment_name}" == "expEuler" ] && [ "${sacado}" == "sacado" ]; then + continue + fi + export sflag=$sacado + run_this="cd $dir;./$exe;cd -" + echo $run_this + eval $run_this + done + done + echo "end of outer loop" +done diff --git a/scripts/aerosol_performance/runThisCPU.sh b/scripts/aerosol_performance/runThisCPU.sh new file mode 100644 index 00000000..f554ad62 --- /dev/null +++ b/scripts/aerosol_performance/runThisCPU.sh @@ -0,0 +1,31 @@ +for number_of_particles in ${nparticles[@]}; do +for N in ${nbatch[@]}; do + for threads in ${Nthread[@]}; do + echo "nbatch = $N" + echo "Nthread = $threads" + thread_size=${threads} + output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" + output="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.txt" + echo "${tchem_outputs}" + echo "${experiment_name}" + echo "/reaction_rates_nbatch_${N}_thread_size_${thread_size}.txt" + echo "output file = " $output + run_this="OMP_NUM_THREADS=$thread_size OMP_PLACES=threads OMP_PROC_BIND=close ${exec} \ + --batch_size=$N \ + --use_cloned_samples=$use_cloned_samples \ + --verbose=$verbose \ + --outputfile_times=$output_wall_times \ + --outputfile=$output \ + --number_of_particles=$number_of_particles \ + $scenario_n_inputs \ + $numerical_params " + echo $run_this + eval $run_this + sleep 2 + # Note: we use this with kokkos tools + kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_thread_size_${thread_size}_number_of_particles${number_of_particles}.json" + sleep 2 + rm -rf $machine_name* + done +done +done diff --git a/scripts/aerosol_performance/runThisGPU.sh b/scripts/aerosol_performance/runThisGPU.sh new file mode 100644 index 00000000..bc9654a0 --- /dev/null +++ b/scripts/aerosol_performance/runThisGPU.sh @@ -0,0 +1,33 @@ +for number_of_particles in ${nparticles[@]}; do +for N in ${nbatch[@]}; do + for i in "${!vector_size[@]}"; do + vector_thread_size=${vector_size[i]} + team_thread_size=${team_size[i]} + echo "nbatch = $N" + echo "vector thread size = $vector_thread_size" + echo "team thread size = $team_thread_size" + output_wall_times="${tchem_outputs}/${experiment_name}/wall_times_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}_number_of_particles${number_of_particles}.json" + output_file="${tchem_outputs}/${experiment_name}/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}_number_of_particles${number_of_particles}.txt" + echo "${tchem_outputs}" + echo "${experiment_name}" + echo "/reaction_rates_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}.txt" + echo "output file = ${output_file}" + run_this="OMP_NUM_THREADS=1 OMP_PLACES=threads OMP_PROC_BIND=close $exec \ + --batch_size=$N \ + --team_thread_size=$vector_thread_size \ + --vector_thread_size=$vector_thread_size \ + --use_cloned_samples=$use_cloned_samples \ + --verbose=$verbose \ + --outputfile_times=$output_wall_times \ + --outputfile=$output_file \ + $scenario_n_inputs \ + $numerical_params " + echo $run_this + eval $run_this + sleep 2 + kp_json_writer $machine_name* > "${tchem_outputs}/${experiment_name}/simple_timer_nbatch_${N}_vecsize_${vector_thread_size}_teamThread_size_${team_thread_size}_number_of_particles${number_of_particles}.json" + sleep 2 + rm -rf $machine_name* +done +done +done diff --git a/scripts/build_kokkostools.sh b/scripts/build_kokkostools.sh new file mode 100755 index 00000000..71255c09 --- /dev/null +++ b/scripts/build_kokkostools.sh @@ -0,0 +1,37 @@ + +MY_CC=gcc +MY_CXX=g++ +JFLAG="-j 10" + +REPO_BASE=${PWD}/kokkos-tools +BUILD_BASE=${PWD}/kokkos-tools +INSTALL_BASE=${PWD}/kokkos-tools + +get_kokkostools (){ +echo "get kokkostools:" +if [ -d "${KOKKOSTOOLS_REPOSITORY_PATH}" ] && [ "$(ls -A ${KOKKOSTOOLS_REPOSITORY_PATH})" ]; then + echo "${KOKKOSTOOLS_REPOSITORY_PATH} exists and is not empty ... aborting clone"; return +fi +git clone https://github.com/kokkos/kokkos-tools.git ${KOKKOSTOOLS_REPOSITORY_PATH} +} + +build_install_kokkostools(){ +echo "Building kokkos tools:" +mkdir ${KOKKOSTOOLS_BUILD_PATH} +cd ${KOKKOSTOOLS_BUILD_PATH} +cmake \ + -D CMAKE_INSTALL_PREFIX=${KOKKOSTOOLS_INSTALL_PATH} \ + -D CMAKE_CXX_COMPILER="${MY_CXX}" \ + -D CMAKE_C_COMPILER="${MY_CC}" \ + -D CMAKE_BUILD_TYPE=RELEASE \ + ${KOKKOSTOOLS_REPOSITORY_PATH} +make ${JFLAG} install +} +KOKKOSTOOLS_REPOSITORY_PATH=${REPO_BASE}/main +KOKKOSTOOLS_BUILD_PATH=${BUILD_BASE}/build +KOKKOSTOOLS_INSTALL_PATH=${INSTALL_BASE}/install + +get_kokkostools +build_install_kokkostools + +exit diff --git a/src/core/TChem_AerosolModelData.cpp b/src/core/TChem_AerosolModelData.cpp index 59e0e9ec..a8e6fde3 100644 --- a/src/core/TChem_AerosolModelData.cpp +++ b/src/core/TChem_AerosolModelData.cpp @@ -225,6 +225,17 @@ int AerosolModelData::initChem(YAML::Node &root, return 0; } +void AerosolModelData::setNumberofParticles(const ordinal_type number_of_particles) +{ + printf("-------------------------------------------------------\n"); + printf("--------------------Warning----------------------------\n"); + printf("Setting number of particles\n"); + printf("Old value : %d \n", nParticles_); + nParticles_=number_of_particles; + printf("Current value : %d \n", nParticles_); + printf("-------------------------------------------------------\n"); +} + void AerosolModelData::scenarioConditionParticles(const std::string &mechfile, const ordinal_type nBatch, real_type_2d_view_host& num_concentration, diff --git a/src/core/TChem_AerosolModelData.hpp b/src/core/TChem_AerosolModelData.hpp index 3cc9ded8..6e6678e0 100644 --- a/src/core/TChem_AerosolModelData.hpp +++ b/src/core/TChem_AerosolModelData.hpp @@ -37,7 +37,7 @@ namespace TChem { ordinal_type nSpec_gas_; ordinal_type nConstSpec_gas_; // number of particles - ordinal_type nParticles_; + ordinal_type nParticles_{-1}; // aerosol molecular weights and density real_type_1d_dual_view molecular_weights_, aerosol_density_; simpol_phase_transfer_type_1d_dual_view simpol_params_; @@ -64,6 +64,8 @@ namespace TChem { void initFile(const std::string &mechfile, std::ostream& echofile); ordinal_type initChem(YAML::Node& doc, std::ostream& echofile); + void setNumberofParticles(const ordinal_type number_of_particles); + void setGasParameters(const KineticModelData& kmd); void scenarioConditionParticles(const std::string &mechfile, const ordinal_type nBatch, diff --git a/src/core/TChem_AtmosphericChemistry.cpp b/src/core/TChem_AtmosphericChemistry.cpp index f4eb1fbb..9a1076ae 100644 --- a/src/core/TChem_AtmosphericChemistry.cpp +++ b/src/core/TChem_AtmosphericChemistry.cpp @@ -539,9 +539,9 @@ TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(const std::string& if (nBatch_photo_reation == nBatch) { // std::cout << photo_rate_values << "\n"; - for (int ibacth = 0; ibacth < nBatch; ++ibacth) + for (int ibatch = 0; ibatch < nBatch; ++ibatch) { - photo_rates_host(ibacth, ireac) = photo_rate_values[ibacth].as(); + photo_rates_host(ibatch, ireac) = photo_rate_values[ibatch].as(); } } else { printf("Error number of values in photo reaction is different than number of conditions"); @@ -610,9 +610,9 @@ if (root["external_forcing"]){ if (nBatch_ext_forcing == nBatch) { - for (int ibacth = 0; ibacth < nBatch; ++ibacth) + for (int ibatch = 0; ibatch < nBatch; ++ibatch) { - external_forcing_host(ibacth, sp_idx) = ext_forcing_values[ibacth].as(); + external_forcing_host(ibatch, sp_idx) = ext_forcing_values[ibatch].as(); } } else { printf("Error number of values in external forcing is different than number of conditions"); diff --git a/src/core/impl/TChem_Impl_Aerosol_RHS.hpp b/src/core/impl/TChem_Impl_Aerosol_RHS.hpp index 9fa67ba1..ba834da6 100644 --- a/src/core/impl/TChem_Impl_Aerosol_RHS.hpp +++ b/src/core/impl/TChem_Impl_Aerosol_RHS.hpp @@ -84,10 +84,10 @@ struct Aerosol_RHS work, kmcd); // 2. update RHS of gas and aerosol species - member.team_barrier(); + // member.team_barrier(); using SIMPOL_single_particle_type = TChem::Impl::SIMPOL_single_particle; Kokkos::parallel_for( - Kokkos::TeamThreadRange(member, amcd.nParticles), + Kokkos::ThreadVectorRange(member, amcd.nParticles), [&](const ordinal_type& i_part) { for (size_t i_simpol = 0; i_simpol < amcd.nSimpol_tran; i_simpol++) { diff --git a/src/examples/CMakeLists.txt b/src/examples/CMakeLists.txt index 4e3f811a..0e3580d1 100644 --- a/src/examples/CMakeLists.txt +++ b/src/examples/CMakeLists.txt @@ -13,6 +13,7 @@ LIST(APPEND TCHEM_ATM_EXAMPLE_SOURCES TChem_AtmosphericChemistryE3SM_CVODE.cpp TChem_AerosolChemistry_CVODE.cpp TChem_AerosolChemistry.cpp + TChem_AerosolChemistry_RHSs.cpp ) IF(TCHEM_ATM_ENABLE_TPL_KOKKOSKERNELS) LIST(APPEND TCHEM_ATM_EXAMPLE_SOURCES @@ -72,4 +73,4 @@ TARGET_LINK_LIBRARIES(${TCHEM_ATM_EXAMPLE_EXE} ${TCHEM_ATM_LINK_LIBRARIES}) INSTALL(TARGETS ${TCHEM_ATM_EXAMPLE_EXE} PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE DESTINATION "${CMAKE_INSTALL_PREFIX}/${TCHEM_ATM_INSTALL_EXAMPLE_PATH}") -ENDIF() \ No newline at end of file +ENDIF() diff --git a/src/examples/TChem_AerosolChemistry.cpp b/src/examples/TChem_AerosolChemistry.cpp index 55586952..b984904a 100644 --- a/src/examples/TChem_AerosolChemistry.cpp +++ b/src/examples/TChem_AerosolChemistry.cpp @@ -61,6 +61,10 @@ int main(int argc, char *argv[]) { std::string inputFile("None"); std::string input_file_particles("None"); bool use_cloned_samples(false); + // For scaling studies, we must execute this example many times. + // Thus, we do not want to write the solution to a file. + // In those cases, we pass write_time_profiles false. + bool write_time_profiles(true); /// parse command line arguments TChem::CommandLineParser opts( "This example computes the solution of gas chemistry problem"); @@ -103,6 +107,8 @@ int main(int argc, char *argv[]) { &max_num_newton_iterations); opts.set_option( "verbose", "If true, printout the first Jacobian values", &verbose); + opts.set_option( + "write-time-profiles", "If true, this example will write the time profile output to a file.", &write_time_profiles); opts.set_option("team_thread_size", "time thread size ", &team_size); opts.set_option("batch_size", " number of batches or samples, e.g. 10 ", &nBatch); opts.set_option("vector_thread_size", "vector thread size ", @@ -128,11 +134,11 @@ int main(int argc, char *argv[]) { input_file_particles=chemFile; } -#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) +#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) printf(" TChem is running ATM model box with KokkosKernels\n"); -#else +#else printf(" TChem is running ATM model box with Tines\n"); -#endif +#endif TChem::exec_space().print_configuration(std::cout, detail); TChem::host_exec_space().print_configuration(std::cout, detail); @@ -160,28 +166,31 @@ int main(int argc, char *argv[]) { const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); - FILE *fout = fopen(outputFile.c_str(), "w"); + FILE *fout; + if (write_time_profiles) { + fout = fopen(outputFile.c_str(), "w"); + } // read scenario condition from yaml file // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=1; + ordinal_type nbatch_files=1; printf("conditions parsing ...\n"); TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); printf("Number of const species %d \n", kmcd.nConstSpec); real_type_2d_view_host num_concentration_host; real_type_2d_view_host state_host; - + // scenario particles - amd.scenarioConditionParticles(input_file_particles, nbacth_files, num_concentration_host, state_scenario_host); - + amd.scenarioConditionParticles(input_file_particles, nbatch_files, num_concentration_host, state_scenario_host); + real_type_2d_view num_concentration; real_type_2d_view state; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -191,7 +200,7 @@ int main(int argc, char *argv[]) { auto state_at_0 = Kokkos::subview(state, 0, Kokkos::ALL); Kokkos::deep_copy(state_at_0, state_scenario_host_at_0); TChem::Test::cloneView(state); - //make sure to copy state to host view. + //make sure to copy state to host view. Kokkos::deep_copy(state_host, state); //scenario particles auto num_concentration_host_at_0 = Kokkos::subview(num_concentration_host, 0, Kokkos::ALL); @@ -201,16 +210,16 @@ int main(int argc, char *argv[]) { TChem::Test::cloneView(num_concentration); } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); state_host = state_scenario_host; - // scenario particles + // scenario particles num_concentration = real_type_2d_view("num_concentration", nBatch, amd.nParticles_); Kokkos::deep_copy(num_concentration, num_concentration_host); } - printf("Number of nbacth %d \n",nBatch); + printf("Number of nbatch %d \n",nBatch); auto writeState = [](const ordinal_type iter, const real_type_1d_view_host _t, const real_type_1d_view_host _dt, @@ -238,14 +247,14 @@ int main(int argc, char *argv[]) { using policy_type = typename TChem::UseThisTeamPolicy::type; -#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) +#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) /// team policy // The Kokkos-kernels BDF solver is designed for range policy. Therefore, I will use a team size of 1. policy_type policy(exec_space_instance, nBatch, 1); -#else +#else policy_type policy(exec_space_instance, nBatch, Kokkos::AUTO()); -#endif +#endif ordinal_type number_of_equations(0); @@ -258,11 +267,11 @@ int main(int argc, char *argv[]) { const ordinal_type level = 1; ordinal_type per_team_extent(0); -#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) +#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) per_team_extent = TChem::AerosolChemistry_KokkosKernels::getWorkSpaceSize(kmcd, amcd); #else per_team_extent = TChem::AerosolChemistry::getWorkSpaceSize(kmcd, amcd); -#endif +#endif const ordinal_type per_team_scratch = TChem::Scratch::shmem_size(per_team_extent); policy.set_scratch_size(level, Kokkos::PerTeam(per_team_scratch)); @@ -314,35 +323,33 @@ int main(int argc, char *argv[]) { dt_host = real_type_1d_view_host("dt host", nBatch); ordinal_type iter = 0; - - fprintf(fout, "%s \t %s \t %s \t ", "iter", "t", "dt"); - fprintf(fout, "%s \t %s \t %s \t", "Density[kg/m3]", "Pressure[Pascal]", + if (write_time_profiles) { + fprintf(fout, "%s \t %s \t %s \t ", "iter", "t", "dt"); + fprintf(fout, "%s \t %s \t %s \t", "Density[kg/m3]", "Pressure[Pascal]", "Temperature[K]"); - for (ordinal_type k = 0; k < kmcd.nSpec; k++) - fprintf(fout, "%s \t", &speciesNamesHost(k, 0)); - //given aero idx return species name - std::map aero_idx_sp_name; - for (std::map::iterator - i = amd.aerosol_sp_name_idx_.begin(); - i != amd.aerosol_sp_name_idx_.end(); ++i) - aero_idx_sp_name[i->second] = i->first; + for (ordinal_type k = 0; k < kmcd.nSpec; k++) + fprintf(fout, "%s \t", &speciesNamesHost(k, 0)); + //given aero idx return species name + std::map aero_idx_sp_name; + for (std::map::iterator + i = amd.aerosol_sp_name_idx_.begin(); + i != amd.aerosol_sp_name_idx_.end(); ++i) + aero_idx_sp_name[i->second] = i->first; - for (ordinal_type ipart = 0; ipart < amd.nParticles_; ipart++) - { - for (ordinal_type isp = 0; isp < amd.nSpec_; isp++) + for (ordinal_type ipart = 0; ipart < amd.nParticles_; ipart++) { - // std::cout << "species Name : "<< aero_idx_sp_name[i] << "\n"; - auto aero_sp_name = aero_idx_sp_name[isp]+"_p"+std::to_string(ipart); - fprintf(fout, "%s \t", aero_sp_name.c_str()); - }// isp - }// ipar - - - - fprintf(fout, "\n"); - writeState(-1, t_host, dt_host, state_host, fout); - + for (ordinal_type isp = 0; isp < amd.nSpec_; isp++) + { + // std::cout << "species Name : "<< aero_idx_sp_name[i] << "\n"; + auto aero_sp_name = aero_idx_sp_name[isp]+"_p"+std::to_string(ipart); + fprintf(fout, "%s \t", aero_sp_name.c_str()); + }// isp + }// ipar + + fprintf(fout, "\n"); + writeState(-1, t_host, dt_host, state_host, fout); + } real_type tsum(0); Kokkos::Timer timer; @@ -351,7 +358,7 @@ int main(int argc, char *argv[]) { timer.reset(); -#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) +#if defined(TCHEM_EXAMPLE_KOKKOS_KERNELS_ODE) TChem::AerosolChemistry_KokkosKernels::runDeviceBatch( policy, tol_time, fac, tadv, state, num_concentration, t, dt, state, kmcd, amcd); @@ -367,15 +374,12 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "\"%s%d\": %20.14e, \n", "wall_time_iter_", iter, t_device_batch); - - - Kokkos::deep_copy(dt_host, dt); - Kokkos::deep_copy(t_host, t); - Kokkos::deep_copy(state_host, state); - - - writeState(iter, t_host, dt_host, state_host, fout); - + if (verbose) { + Kokkos::deep_copy(dt_host, dt); + Kokkos::deep_copy(t_host, t); + Kokkos::deep_copy(state_host, state); + writeState(iter, t_host, dt_host, state_host, fout); + } /// carry over time and dt computed in this step tsum = zero; Kokkos::parallel_reduce( @@ -400,8 +404,9 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "%s: %d \n", "\"number_of_samples\"", nBatch); fprintf(fout_times, "} \n "); // reaction rates - - fclose(fout); + if (write_time_profiles) { + fclose(fout); + } fprintf(fout_times, "}\n "); // end index time fclose(fout_times); } diff --git a/src/examples/TChem_AerosolChemistry_CVODE.cpp b/src/examples/TChem_AerosolChemistry_CVODE.cpp index 2da26e72..16246218 100644 --- a/src/examples/TChem_AerosolChemistry_CVODE.cpp +++ b/src/examples/TChem_AerosolChemistry_CVODE.cpp @@ -62,7 +62,10 @@ int main(int argc, char *argv[]) { std::string inputFile("None"); std::string input_file_particles("None"); bool use_cloned_samples(false); - bool use_cvode(false); + // For scaling studies, we must execute this example many times. + // Thus, we do not want to write the solution to a file. + // In those cases, we pass write_time_profiles false. + bool write_time_profiles(true); /// parse command line arguments TChem::CommandLineParser opts( "This example computes the solution of gas chemistry problem"); @@ -105,6 +108,7 @@ int main(int argc, char *argv[]) { &max_num_newton_iterations); opts.set_option( "verbose", "If true, printout the first Jacobian values", &verbose); + opts.set_option("team_thread_size", "time thread size ", &team_size); opts.set_option("batch_size", " number of batches or samples, e.g. 10 ", &nBatch); opts.set_option("vector_thread_size", "vector thread size ", @@ -112,9 +116,8 @@ int main(int argc, char *argv[]) { opts.set_option("atol-time", "Absolute tolerance used for adaptive time stepping", &atol_time); opts.set_option( "use_cloned_samples", "If true, one state vector will be cloned.", &use_cloned_samples); - opts.set_option("use-cvode", - "if true code runs ignition zero d reactor with cvode; otherwise, it uses TrBDF2", - &use_cvode); + opts.set_option( + "write-time-profiles", "If true, this example will write the time profile output to a file.", &write_time_profiles); const bool r_parse = opts.parse(argc, argv); if (r_parse) @@ -163,23 +166,26 @@ int main(int argc, char *argv[]) { const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); - FILE *fout = fopen(outputFile.c_str(), "w"); + FILE *fout; + if (write_time_profiles) { + fout = fopen(outputFile.c_str(), "w"); + } + // read scenario condition from yaml file // read scenario condition from yaml file - // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=1; + ordinal_type nbatch_files=1; printf("conditions parsing ...\n"); TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); const ordinal_type n_active_vars = total_n_species - kmcd.nConstSpec; printf("Number of const species %d \n", kmcd.nConstSpec); real_type_2d_view_host num_concentration_scenario, num_concentration; - amd.scenarioConditionParticles(input_file_particles, nbacth_files, num_concentration_scenario, state_scenario_host); + amd.scenarioConditionParticles(input_file_particles, nbatch_files, num_concentration_scenario, state_scenario_host); real_type_2d_view_host state; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -197,14 +203,14 @@ int main(int argc, char *argv[]) { TChem::Test::cloneView(num_concentration); } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = state_scenario_host; - // scenario particles + // scenario particles num_concentration = real_type_2d_view_host("num_concentration", nBatch, amd.nParticles_); Kokkos::deep_copy(num_concentration, num_concentration_scenario); } - printf("Number of nbacth %d \n",nBatch); + printf("Number of nbatch %d \n",nBatch); auto writeState = [](const ordinal_type iter, const real_type_1d_view_host _t, const real_type_1d_view_host _dt, @@ -248,11 +254,10 @@ int main(int argc, char *argv[]) { using time_integrator_cvode_type = Tines::TimeIntegratorCVODE; Tines::value_type_1d_view cvodes; - if (use_cvode) { - cvodes = Tines::value_type_1d_view("cvodes", nBatch); - for (ordinal_type i=0;i("cvodes", nBatch); + for (ordinal_type i=0;i aero_idx_sp_name; - for (std::map::iterator - i = amd.aerosol_sp_name_idx_.begin(); - i != amd.aerosol_sp_name_idx_.end(); ++i) - aero_idx_sp_name[i->second] = i->first; - - for (ordinal_type ipart = 0; ipart < amd.nParticles_; ipart++) - { - for (ordinal_type isp = 0; isp < amd.nSpec_; isp++) + for (ordinal_type k = 0; k < kmcd.nSpec; k++) + fprintf(fout, "%s \t", &speciesNamesHost(k, 0)); + //given aero idx return species name + std::map aero_idx_sp_name; + for (std::map::iterator + i = amd.aerosol_sp_name_idx_.begin(); + i != amd.aerosol_sp_name_idx_.end(); ++i) + aero_idx_sp_name[i->second] = i->first; + for (ordinal_type ipart = 0; ipart < amd.nParticles_; ipart++) { - // std::cout << "species Name : "<< aero_idx_sp_name[i] << "\n"; - auto aero_sp_name = aero_idx_sp_name[isp]+"_p"+std::to_string(ipart); - fprintf(fout, "%s \t", aero_sp_name.c_str()); - }// isp - }// ipar - - - - fprintf(fout, "\n"); - writeState(-1, t, dt, state, fout); + for (ordinal_type isp = 0; isp < amd.nSpec_; isp++) + { + // std::cout << "species Name : "<< aero_idx_sp_name[i] << "\n"; + auto aero_sp_name = aero_idx_sp_name[isp]+"_p"+std::to_string(ipart); + fprintf(fout, "%s \t", aero_sp_name.c_str()); + }// isp + }// ipar + + fprintf(fout, "\n"); + writeState(-1, t, dt, state, fout); + }// write_time_profiles real_type tsum(0); Kokkos::Timer timer; @@ -355,8 +359,9 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "\"%s%d\": %20.14e, \n", "wall_time_iter_", iter, t_device_batch); - writeState(iter, t, dt, state, fout); - + if (write_time_profiles) { + writeState(iter, t, dt, state, fout); + } /// carry over time and dt computed in this step tsum = zero; Kokkos::parallel_reduce( @@ -381,8 +386,9 @@ int main(int argc, char *argv[]) { fprintf(fout_times, "%s: %d \n", "\"number_of_samples\"", nBatch); fprintf(fout_times, "} \n "); // reaction rates - - fclose(fout); + if (write_time_profiles) { + fclose(fout); + } fprintf(fout_times, "}\n "); // end index time fclose(fout_times); } diff --git a/src/examples/TChem_AerosolChemistry_RHSs.cpp b/src/examples/TChem_AerosolChemistry_RHSs.cpp new file mode 100644 index 00000000..010a6ae2 --- /dev/null +++ b/src/examples/TChem_AerosolChemistry_RHSs.cpp @@ -0,0 +1,390 @@ +/* ===================================================================================== +TChem-atm version 1.0 +Copyright (2024) NTESS +https://github.com/sandialabs/TChem-atm + +Copyright 2024 National Technology & Engineering Solutions of Sandia, LLC +(NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +Government retains certain rights in this software. + +This file is part of TChem-atm. TChem-atm is open source software: you can redistribute +it and/or modify it under the terms of BSD 2-Clause License +(https://opensource.org/licenses/BSD-2-Clause). A copy of the licese is also +provided under the main directory + +Questions? Contact Oscar Diaz-Ibarra at , or + Mike Schmidt at , or + Cosmin Safta at + +Sandia National Laboratories, New Mexico/Livermore, NM/CA, USA +===================================================================================== +*/ +#include "TChem.hpp" +#include "TChem_CommandLineParser.hpp" +#include "TChem_Impl_AerosolChemistry.hpp" + +using ordinal_type = TChem::ordinal_type; +using real_type = TChem::real_type; + +int main(int argc, char *argv[]) { + std::string chemFile( "chem.yaml"); + std::string aeroFile("aero.yaml"); + std::string outputFile("AerosolChemistry.dat"); + std::string inputFile("None"); + std::string input_file_particles("None"); + std::string outputFileTimes("wall_times.json"); + int nBatch(1), team_size(-1), vector_size(-1); + bool verbose(true); + bool use_cloned_samples(false); + int number_of_particles(-1); + + /// parse command line arguments + TChem::CommandLineParser opts("This example computes the net production rates with a given state vector"); + opts.set_option("chemfile", "Chem file name e.g., chem.inp", + &chemFile); + opts.set_option("aerofile", "Aerosol Chem file name e.g., aero.yaml", + &aeroFile); + opts.set_option("inputfile", "Chem file name e.g., chem.inp", + &inputFile); + opts.set_option("inputfile_particles", "input file name e.g., chem.inp", + &input_file_particles); + opts.set_option("outputfile", "Output omega file name e.g., omega.dat", &outputFile); + opts.set_option("outputfile_times", "Wal times file name e.g., times.json", &outputFileTimes); + opts.set_option("team_thread_size", "time thread size ", &team_size); + opts.set_option("vector_thread_size", "vector thread size ", &vector_size); + opts.set_option("number_of_particles", "Set the number of particles; this will overwrite the value from the input file. ", &number_of_particles); + opts.set_option("verbose", "If true, printout the first omega values", &verbose); + opts.set_option("batch_size", " number of batches or samples, e.g. 10 ", &nBatch); + opts.set_option( + "use_cloned_samples", "If true, one state vector will be cloned.", &use_cloned_samples); + + const bool r_parse = opts.parse(argc, argv); + if (r_parse) + return 0; // print help return + + Kokkos::initialize(argc, argv); + { + + // by default use input condition from chemFile + if (inputFile == "None") { + inputFile=chemFile; + } + // check first in chem file + if (input_file_particles == "None") { + input_file_particles=chemFile; + } + + const bool detail = false; + constexpr real_type zero =0; + + using host_device_type = typename Tines::UseThisDevice::type; + + using device_type = typename Tines::UseThisDevice::type; + using problem_type = TChem::Impl::AerosolChemistry_Problem; + + using real_type_0d_view_type = Tines::value_type_0d_view; + using real_type_1d_view_type = Tines::value_type_1d_view; + using real_type_2d_view_type = Tines::value_type_2d_view; + using real_type_3d_view_type = Tines::value_type_3d_view; + using real_type_2d_view_host_type = Tines::value_type_2d_view; + + + TChem::exec_space().print_configuration(std::cout, detail); + TChem::host_exec_space().print_configuration(std::cout, detail); + const auto exec_space_instance = TChem::exec_space(); + + /// construct kmd and use the view for testing + printf("kmd parsing ...\n"); + TChem::KineticModelData kmd(chemFile); + const auto kmcd = TChem::createNCAR_KineticModelConstData(kmd); + + printf("amd parsing ...\n"); + TChem::AerosolModelData amd(aeroFile, kmd); + if(number_of_particles > 0) { + amd.setNumberofParticles(number_of_particles); + } + const auto amcd = TChem::create_AerosolModelConstData(amd); + + const ordinal_type total_n_species =kmcd.nSpec + amcd.nSpec * amcd.nParticles; + const ordinal_type number_of_equations = problem_type::getNumberOfTimeODEs(kmcd, amcd); + const ordinal_type stateVecDim = + TChem::Impl::getStateVectorSize(total_n_species); + const auto speciesNamesHost = Kokkos::create_mirror_view(kmcd.speciesNames); + Kokkos::deep_copy(speciesNamesHost, kmcd.speciesNames); + + printf("Number of Gas Species %d \n", kmcd.nSpec); + printf("Number of Gas Reactions %d \n", kmcd.nReac); + + printf("Number of Aerosol Species %d \n", amcd.nSpec); + printf("Number of Aerosol Particles %d \n", amcd.nParticles); + + printf("stateVecDim %d \n", stateVecDim); + + // read scenario condition from yaml file + real_type_2d_view_host_type state_scenario_host; + ordinal_type nbatch_files=1; + printf("conditions parsing ...\n"); + TChem::AtmChemistry::setScenarioConditions(inputFile, + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); + printf("Number of const species %d \n", kmcd.nConstSpec); + + real_type_2d_view_host_type num_concentration_host; + real_type_2d_view_host_type state_host; + + // scenario particles + amd.scenarioConditionParticles(input_file_particles, + nbatch_files, + num_concentration_host, + state_scenario_host); + + real_type_2d_view_type num_concentration; + real_type_2d_view_type state; + + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 + printf("-------------------------------------------------------\n"); + printf("--------------------Warning----------------------------\n"); + printf("Using cloned samples ... only for numerical experiments\n"); + state = real_type_2d_view_type("StateVector Devices", nBatch, stateVecDim); + state_host = Kokkos::create_mirror_view(state); + auto state_scenario_host_at_0 = Kokkos::subview(state_scenario_host, 0, Kokkos::ALL); + auto state_at_0 = Kokkos::subview(state, 0, Kokkos::ALL); + Kokkos::deep_copy(state_at_0, state_scenario_host_at_0); + TChem::Test::cloneView(state); + //make sure to copy state to host view. + Kokkos::deep_copy(state_host, state); + //scenario particles + auto num_concentration_host_at_0 = Kokkos::subview(num_concentration_host, 0, Kokkos::ALL); + num_concentration = real_type_2d_view_type("num_concentration", nBatch, amd.nParticles_); + auto num_concentration_at_0 = Kokkos::subview(num_concentration, 0, Kokkos::ALL); + Kokkos::deep_copy(num_concentration_at_0, num_concentration_host_at_0); + TChem::Test::cloneView(num_concentration); + + } else { + nBatch = nbatch_files; + state = real_type_2d_view_type("StateVector Devices", nBatch, stateVecDim); + Kokkos::deep_copy(state, state_scenario_host); + state_host = state_scenario_host; + + // scenario particles + num_concentration = real_type_2d_view_type("num_concentration", nBatch, amd.nParticles_); + Kokkos::deep_copy(num_concentration, num_concentration_host); + } + // output + real_type_2d_view_type rhs("rhs", nBatch, number_of_equations); + real_type_2d_view_type fac("fac", nBatch, number_of_equations); + real_type_3d_view_type jacobian("jacobian", nBatch, number_of_equations, number_of_equations); + + using policy_type = typename TChem::UseThisTeamPolicy::type; + + // fprintf(fout_times, " Computation type, total time [sec], time per sample [sec/sample]\n "); + + policy_type policy(exec_space_instance, nBatch, Kokkos::AUTO()); + + if (team_size > 0 && vector_size > 0) { + policy = policy_type(exec_space_instance, nBatch, team_size, vector_size); + } else if (team_size > 0 && vector_size < 0) { + // only set team size + policy = policy_type(exec_space_instance, nBatch, team_size); + } + + using range_type = Kokkos::pair; + const ordinal_type level = 1; + const ordinal_type per_team_extent = problem_type::getWorkSpaceSize(kmcd,amcd) + + number_of_equations; + const ordinal_type per_team_scratch = + TChem::Scratch::shmem_size(per_team_extent); + policy.set_scratch_size(level, Kokkos::PerTeam(per_team_scratch)); + + Kokkos::Timer timer; + FILE *fout_times = fopen(outputFileTimes.c_str(), "w"); + { + fprintf(fout_times, "{\n"); + fprintf(fout_times, " \"Aerosol RHSs\": \n {\n"); + const ordinal_type level = 1; + const std::string profile_name = "TChem::AerosolChemistry::RHS_evaluation"; + Kokkos::fence(); + timer.reset(); + Kokkos::Profiling::pushRegion(profile_name); + Kokkos::parallel_for + (profile_name, + policy, + KOKKOS_LAMBDA(const typename policy_type::member_type& member) { + const ordinal_type i = member.league_rank(); + + const ordinal_type m = problem_type::getNumberOfTimeODEs(kmcd,amcd); + const real_type_1d_view_type rhs_at_i = + Kokkos::subview(rhs, i, Kokkos::ALL()); + + const real_type_1d_view_type state_at_i = + Kokkos::subview(state, i, Kokkos::ALL()); + + const real_type_1d_view_type number_conc_at_i = + Kokkos::subview(num_concentration, i, Kokkos::ALL()); + TChem::Scratch work(member.team_scratch(level), + per_team_extent); + auto wptr = work.data(); + const ordinal_type total_n_species = kmcd.nSpec + amcd.nParticles*amcd.nSpec; + TChem::Impl::StateVector sv_at_i(total_n_species, state_at_i); + + const real_type temperature = sv_at_i.Temperature(); + const real_type pressure = sv_at_i.Pressure(); + const real_type_1d_view_type Ys = sv_at_i.MassFractions(); + const ordinal_type n_active_gas_species = kmcd.nSpec - kmcd.nConstSpec; + const auto activeYs = real_type_1d_view_type(Ys.data(), + n_active_gas_species ); + const auto constYs = real_type_1d_view_type(Ys.data(), + +n_active_gas_species, kmcd.nSpec ); + const real_type_1d_view_type partYs = Kokkos::subview(Ys, range_type(kmcd.nSpec, total_n_species)); + + real_type_1d_view_type vals(wptr, m); + wptr += m; + + /// problem workspace + /// problem setup + const ordinal_type problem_workspace_size = problem_type::getWorkSpaceSize(kmcd,amcd); + auto pw = real_type_1d_view_type(wptr, problem_workspace_size); + wptr +=problem_workspace_size; + problem_type problem; + problem._kmcd = kmcd; + problem._amcd = amcd; + + /// initialize problem + // problem._fac = fac_at_i; + problem._work = pw; + problem._temperature= temperature; + problem._pressure =pressure; + problem._const_concentration= constYs; + problem._number_conc =number_conc_at_i; + // active gas species + for (ordinal_type i=0;i work(member.team_scratch(level), + per_team_extent); + auto wptr = work.data(); + const ordinal_type total_n_species = kmcd.nSpec + amcd.nParticles*amcd.nSpec; + TChem::Impl::StateVector sv_at_i(total_n_species, state_at_i); + + const real_type temperature = sv_at_i.Temperature(); + const real_type pressure = sv_at_i.Pressure(); + const real_type_1d_view_type Ys = sv_at_i.MassFractions(); + const ordinal_type n_active_gas_species = kmcd.nSpec - kmcd.nConstSpec; + const auto activeYs = real_type_1d_view_type(Ys.data(), + n_active_gas_species ); + const auto constYs = real_type_1d_view_type(Ys.data(), + +n_active_gas_species, kmcd.nSpec ); + const real_type_1d_view_type partYs = Kokkos::subview(Ys, range_type(kmcd.nSpec, total_n_species)); + + real_type_1d_view_type vals(wptr, m); + wptr += m; + + /// problem workspace + /// problem setup + const ordinal_type problem_workspace_size = problem_type::getWorkSpaceSize(kmcd,amcd); + auto pw = real_type_1d_view_type(wptr, problem_workspace_size); + wptr +=problem_workspace_size; + problem_type problem; + problem._kmcd = kmcd; + problem._amcd = amcd; + + /// initialize problem + problem._fac = fac_at_i; + problem._work = pw; + problem._temperature= temperature; + problem._pressure =pressure; + problem._const_concentration= constYs; + problem._number_conc =number_conc_at_i; + // active gas species + for (ordinal_type i=0;i(0, nBatch), KOKKOS_LAMBDA(const ordinal_type &i) { printf("Devices::Initial condition sample No %d\n", i); @@ -139,7 +140,7 @@ int main(int argc, char *argv[]) { printf(" %e", state_at_i(k)); printf("\n"); }); - + } // auto writeState = [](const ordinal_type iter, const real_type_1d_view_host _t, const real_type_1d_view_host _dt, @@ -157,13 +158,13 @@ int main(int argc, char *argv[]) { }; - auto printState = [](const time_advance_type _tadv, const real_type _t, const real_type_1d_view_host _state_at_i) { - /// iter, t, dt, rho, pres, temp, Ys .... + auto printState = [](const time_advance_type _tadv, const real_type _t, const real_type_1d_view_host _state_at_i) + { printf("%e %e %e %e %e", _t, _t - _tadv._tbeg, _state_at_i(0), _state_at_i(1), _state_at_i(2)); for (ordinal_type k = 3, kend = _state_at_i.extent(0); k < kend; ++k) printf(" %e", _state_at_i(k)); printf("\n"); - + /// iter, t, dt, rho, pres, temp, Ys ... }; timer.reset(); @@ -253,6 +254,7 @@ int main(int argc, char *argv[]) { ordinal_type iter = 0; /// print of store QOI for the first sample + if (verbose) { /// could use cuda streams Kokkos::deep_copy(tadv_at_i_host, tadv_at_i); @@ -278,7 +280,7 @@ int main(int argc, char *argv[]) { TChem::AtmosphericChemistry::runDeviceBatch(policy, tol_newton, tol_time, fac, tadv, state, t, dt, state, kmcd); - { + if (verbose) { /// could use cuda streams Kokkos::deep_copy(tadv_at_i_host, tadv_at_i); Kokkos::deep_copy(t_at_i_host, t_at_i); @@ -314,6 +316,9 @@ int main(int argc, char *argv[]) { const real_type t_device_batch = timer.seconds(); printf("Time ignition %e [sec] %e [sec/sample]\n", t_device_batch, t_device_batch / real_type(nBatch)); + + if (verbose) + { Kokkos::parallel_for( Kokkos::RangePolicy(0, nBatch), KOKKOS_LAMBDA(const ordinal_type &i) { printf("Devices:: Solution sample No %d\n", i); @@ -322,6 +327,7 @@ int main(int argc, char *argv[]) { printf(" %e", state_at_i(k)); printf("\n"); }); + } fclose(fout); diff --git a/src/examples/TChem_AtmosphericChemistryE3SM.cpp b/src/examples/TChem_AtmosphericChemistryE3SM.cpp index 76b8e161..0b385204 100644 --- a/src/examples/TChem_AtmosphericChemistryE3SM.cpp +++ b/src/examples/TChem_AtmosphericChemistryE3SM.cpp @@ -153,16 +153,16 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec,stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec,stateVecDim, state_scenario_host, nbatch_files); // read photolysis reaction values // we assume photolysis reaction are computed by another tool. real_type_2d_view_host photo_rates_scenario_host; ordinal_type n_photo_rates = 0; TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(inputFile, - nbacth_files, + nbatch_files, // output photo_rates_scenario_host, n_photo_rates @@ -174,7 +174,7 @@ int main(int argc, char *argv[]) { printf("Number of species %d \n", kmcd.nSpec); printf("Number of const species %d \n", kmcd.nConstSpec); // Note: We allocate external_sources_scenario_host - real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbacth_files,n_active_vars); + real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbatch_files,n_active_vars); TChem::AtmChemistry::setScenarioConditionsExternalForcing(inputFile, speciesNamesHost, // output @@ -186,8 +186,8 @@ int main(int argc, char *argv[]) { real_type_2d_view_host state_host; real_type_2d_view external_sources; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -224,7 +224,7 @@ int main(int argc, char *argv[]) { } } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); state_host = state_scenario_host; diff --git a/src/examples/TChem_AtmosphericChemistryE3SM_CVODE.cpp b/src/examples/TChem_AtmosphericChemistryE3SM_CVODE.cpp index 12cb98c7..a2575b82 100644 --- a/src/examples/TChem_AtmosphericChemistryE3SM_CVODE.cpp +++ b/src/examples/TChem_AtmosphericChemistryE3SM_CVODE.cpp @@ -152,16 +152,16 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); // read photolysis reaction values // we assume photolysis reaction are computed by another tool. real_type_2d_view_host photo_rates_scenario_host; ordinal_type n_photo_rates = 0; TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(inputFile, - nbacth_files, + nbatch_files, // output photo_rates_scenario_host, n_photo_rates @@ -173,7 +173,7 @@ int main(int argc, char *argv[]) { printf("Number of species %d \n", kmcd.nSpec); printf("Number of const species %d \n", kmcd.nConstSpec); // Note: We allocate external_sources_scenario_host - real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbacth_files,n_active_vars); + real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbatch_files,n_active_vars); TChem::AtmChemistry::setScenarioConditionsExternalForcing(inputFile, speciesNamesHost, // output @@ -184,8 +184,8 @@ int main(int argc, char *argv[]) { real_type_2d_view_host photo_rates; real_type_2d_view_host external_sources; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -216,7 +216,7 @@ int main(int argc, char *argv[]) { } } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = state_scenario_host; if (n_photo_rates > 0 ) diff --git a/src/examples/TChem_Linozv3_StratosphereChemSolver.cpp b/src/examples/TChem_Linozv3_StratosphereChemSolver.cpp index 6571de82..68e2cca1 100644 --- a/src/examples/TChem_Linozv3_StratosphereChemSolver.cpp +++ b/src/examples/TChem_Linozv3_StratosphereChemSolver.cpp @@ -373,7 +373,7 @@ void linv3_stratosphere_solver_test( const int team_size, if (team_size > 0 && vector_size > 0) { policy = policy_type(exec_space_instance, nBatch, team_size, vector_size); } else if (team_size > 0 && vector_size < 0) { - // only set team size + // only set team size policy = policy_type(exec_space_instance, nBatch, team_size); } diff --git a/src/examples/TChem_NetProductionRates.cpp b/src/examples/TChem_NetProductionRates.cpp index f2517cd1..737b1376 100644 --- a/src/examples/TChem_NetProductionRates.cpp +++ b/src/examples/TChem_NetProductionRates.cpp @@ -82,9 +82,9 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); // read photolysis reaction values @@ -92,7 +92,7 @@ int main(int argc, char *argv[]) { real_type_2d_view_host photo_rates_scenario_host; ordinal_type n_photo_rates = 0; TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(inputFile, - nbacth_files, + nbatch_files, // output photo_rates_scenario_host, n_photo_rates @@ -105,7 +105,7 @@ int main(int argc, char *argv[]) { printf("Number of species %d \n", kmcd.nSpec); printf("Number of const species %d \n", kmcd.nConstSpec); // Note: We allocate external_sources_scenario_host - real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbacth_files,n_active_vars); + real_type_2d_view_host external_sources_scenario_host("external_sources_host",nbatch_files,n_active_vars); TChem::AtmChemistry::setScenarioConditionsExternalForcing(inputFile, speciesNamesHost, // output @@ -116,8 +116,8 @@ int main(int argc, char *argv[]) { real_type_2d_view photo_rates; real_type_2d_view external_sources; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -155,13 +155,13 @@ int main(int argc, char *argv[]) { } } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); if (n_photo_rates > 0 ) { - nBatch = nbacth_files; + nBatch = nbatch_files; photo_rates = real_type_2d_view("StateVector Devices", nBatch, n_photo_rates); Kokkos::deep_copy(photo_rates, photo_rates_scenario_host); diff --git a/src/examples/TChem_RateofProgress.cpp b/src/examples/TChem_RateofProgress.cpp index 70ed9975..7c1210ce 100644 --- a/src/examples/TChem_RateofProgress.cpp +++ b/src/examples/TChem_RateofProgress.cpp @@ -84,16 +84,16 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(inputFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); // read photolysis reaction values // we assume photolysis reaction are computed by another tool. real_type_2d_view_host photo_rates_scenario_host; ordinal_type n_photo_rates = 0; TChem::AtmChemistry::setScenarioConditionsPhotolysisReactions(inputFile, - nbacth_files, + nbatch_files, // output photo_rates_scenario_host, n_photo_rates @@ -102,8 +102,8 @@ int main(int argc, char *argv[]) { real_type_2d_view state; real_type_2d_view photo_rates; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -127,13 +127,13 @@ int main(int argc, char *argv[]) { } // n_photo_rates } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); if (n_photo_rates > 0 ) { - nBatch = nbacth_files; + nBatch = nbatch_files; photo_rates = real_type_2d_view("StateVector Devices", nBatch, n_photo_rates); Kokkos::deep_copy(photo_rates, photo_rates_scenario_host); diff --git a/src/examples/TChem_ReactionRates.cpp b/src/examples/TChem_ReactionRates.cpp index 8e9d03db..65808537 100644 --- a/src/examples/TChem_ReactionRates.cpp +++ b/src/examples/TChem_ReactionRates.cpp @@ -78,14 +78,14 @@ int main(int argc, char *argv[]) { // read scenario condition from yaml file real_type_2d_view_host state_scenario_host; - ordinal_type nbacth_files=0; + ordinal_type nbatch_files=0; TChem::AtmChemistry::setScenarioConditions(chemFile, - speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbacth_files); + speciesNamesHost, kmcd.nSpec, stateVecDim, state_scenario_host, nbatch_files); real_type_2d_view state; - if (nbacth_files == 1 && use_cloned_samples && nBatch > 1) { - // only clone samples if nbacth_files is 1 + if (nbatch_files == 1 && use_cloned_samples && nBatch > 1) { + // only clone samples if nbatch_files is 1 printf("-------------------------------------------------------\n"); printf("--------------------Warning----------------------------\n"); printf("Using cloned samples ... only for numerical experiments\n"); @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) { TChem::Test::cloneView(state_host); Kokkos::deep_copy(state, state_host); } else { - nBatch = nbacth_files; + nBatch = nbatch_files; state = real_type_2d_view("StateVector Devices", nBatch, stateVecDim); Kokkos::deep_copy(state, state_scenario_host); }