From 7a53b355093e12888a764cdd0f0dee22d0dfe5ca Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Mon, 4 Nov 2024 14:07:13 -0500 Subject: [PATCH 01/83] Make IMFP calculation energy dependent --- Source/Evolve.cpp | 24 ++++++++++++++++++++---- Source/NuLibTable.H | 4 +++- Source/NuLibTableFunctions.H | 27 +++++++++++++++++++++++++++ Source/ReadNuLibTable.cpp | 24 ++---------------------- 4 files changed, 52 insertions(+), 27 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index eb93981..0eeff27 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -167,7 +167,7 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M NuLib_tabulated NuLib_tabulated_obj(alltables_nulib, logrho_nulib, logtemp_nulib, yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib); - + NuLib_energies NuLib_energies_obj(energy_bottom, energy_top); //The following commented loop can be used to print information about each particle in case debug is needed in future. /* @@ -344,9 +344,25 @@ void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const M //--------------------- Values from NuLib table --------------------------- double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); - int idx_group = NULIBVAR(idx_group); - //FIXME: specify neutrino energy using the following: - // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. + int *helperVarsInt_nulib = NuLib_tabulated_obj.get_helperVarsInt_nulib(); + + double *energy_bottom = NuLib_energies_obj.get_energy_bottom_nulib(); + double *energy_top = NuLib_energies_obj.get_energy_top_nulib(); + + double neutrino_energy_erg = p.rdata(PIdx::pupt); //locate energy bin using this. + double neutrino_energy_MeV = neutrino_energy_erg / (1e6*CGSUnitsConst::eV); + + //Decide which energy bin to use (i.e. determine 'idx_group') + int idx_group = -1; + for (int i=0; i= energy_bottom[i] && neutrino_energy_MeV <= energy_top[i]){ + idx_group = i; + break; + } + } + + if(idx_group == -1) assert(0); //abort if energy bin cannot be found. + //printf("Given neutrino energy = %f, selected bin index = %d\n", neutrino_energy_MeV, idx_group); //idx_species = {0 for electron neutrino, 1 for electron antineutrino and 2 for all other heavier ones} //electron neutrino: [0, 0] diff --git a/Source/NuLibTable.H b/Source/NuLibTable.H index 262460e..bbdb71b 100644 --- a/Source/NuLibTable.H +++ b/Source/NuLibTable.H @@ -12,7 +12,7 @@ void ReadNuLibTable(const std::string nulib_table_name); namespace nulib_private { -// table data + // table data extern double *alltables_nulib; //extern double *epstable; extern double *logrho_nulib; @@ -20,6 +20,8 @@ namespace nulib_private { extern double *yes_nulib; extern double *species_nulib; extern double *group_nulib; + extern double *energy_bottom; + extern double *energy_top; #define NULIBVAR(VAR) helperVarsReal_nulib[helperVarsEnumReal_nulib::VAR] //WARNING: If the number of variables is increased here, then memory allocation for helperVarsReal/helperVarsInt pointer in GRHydroX_ReadEOSTable.cxx will also need to be increased. diff --git a/Source/NuLibTableFunctions.H b/Source/NuLibTableFunctions.H index 95f2c19..3ccd5bd 100644 --- a/Source/NuLibTableFunctions.H +++ b/Source/NuLibTableFunctions.H @@ -19,6 +19,11 @@ struct NuLib_tabulated { AMREX_GPU_DEVICE AMREX_GPU_HOST double *get_helperVarsReal_nulib() const { return helperVarsReal_nulib; };//get_helperVarsReal_nulib + + //--------------- get helperVarsInt pointer --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST int *get_helperVarsInt_nulib() const { + return helperVarsInt_nulib; + };//get_helperVarsInt_nulib //--------------- get opacaties --------------------------------------------- @@ -41,4 +46,26 @@ struct NuLib_tabulated { }; //struct NuLib_tabulated + +struct NuLib_energies { + double *energy_bottom, *energy_top; + + //constructor for NuLib_energies + AMREX_GPU_DEVICE AMREX_GPU_HOST NuLib_energies() = default;//Need to keep it + AMREX_GPU_DEVICE AMREX_GPU_HOST NuLib_energies(double *energy_bottom, double *energy_top): + energy_bottom(energy_bottom), energy_top(energy_top) {} + + //--------------- get energy_bottom pointer --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST double *get_energy_bottom_nulib() const { + return energy_bottom; + };//get_energy_bottom_nulib + + //--------------- get energy_top pointer --------------------------------------------- + AMREX_GPU_DEVICE AMREX_GPU_HOST double *get_energy_top_nulib() const { + return energy_top; + };//get_energy_top_nulib + +}; //struct NuLib_energies + + #endif // NULIBTABLEFUCNTIONS_HXX \ No newline at end of file diff --git a/Source/ReadNuLibTable.cpp b/Source/ReadNuLibTable.cpp index bbed857..61870f5 100644 --- a/Source/ReadNuLibTable.cpp +++ b/Source/ReadNuLibTable.cpp @@ -46,6 +46,8 @@ namespace nulib_private { double *yes_nulib; double *species_nulib; //TODO: Get rid of this? double *group_nulib; + double *energy_bottom; + double *energy_top; double *helperVarsReal_nulib; int *helperVarsInt_nulib; } @@ -135,8 +137,6 @@ void ReadNuLibTable(const std::string nulib_table_name) { } //Allocate memory for energy bin determination. - double *energy_bottom; - double *energy_top; if (!(energy_bottom = myManagedArena.allocate(ngroup_) )) { printf("(ReadNuLibTable.cpp) Cannot allocate memory for NuLib table"); assert(0); @@ -201,24 +201,6 @@ void ReadNuLibTable(const std::string nulib_table_name) { //FIXME: FIXME: Make an assert that rho, temp and Ye are uniformally spaced. - //---------------------------- Energy bin determeination -------------------------------- - //FIXME: FIXME: Set from parameter file. - double given_energy = 55.0; //TODO: Is this log or linear in table? - int idx_group_; - - //Decide which energy bin to use (i.e. determine 'idx_group') - for (int i=0; i= energy_bottom[i] && given_energy <= energy_top[i]){ - idx_group_ = i; - break; - } - } - - printf("Given neutrino energy = %f, selected bin index = %d\n", given_energy, idx_group); - myManagedArena.deallocate(energy_bottom, ngroup_); - myManagedArena.deallocate(energy_top, ngroup_); - //---------------------------------------------------------------------------------------------- - //allocate memory for helperVars helperVarsReal_nulib = myManagedArena.allocate(24); helperVarsInt_nulib = myManagedArena_Int.allocate(5); @@ -226,8 +208,6 @@ void ReadNuLibTable(const std::string nulib_table_name) { const double temp0_ = exp(logtemp_nulib[0]); const double temp1_ = exp(logtemp_nulib[1]); - NULIBVAR(idx_group) = idx_group_; - NULIBVAR_INT(nrho) = nrho_; NULIBVAR_INT(ntemp) = ntemp_; NULIBVAR_INT(nye) = nye_; From de0bac5c2354dff1a6127b28f38c18564433653e Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Mon, 4 Nov 2024 15:12:16 -0500 Subject: [PATCH 02/83] Ensure that rho, temp and Ye are uniformally spaced. --- Source/ReadNuLibTable.cpp | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/Source/ReadNuLibTable.cpp b/Source/ReadNuLibTable.cpp index 61870f5..f55e3c3 100644 --- a/Source/ReadNuLibTable.cpp +++ b/Source/ReadNuLibTable.cpp @@ -198,8 +198,7 @@ void ReadNuLibTable(const std::string nulib_table_name) { for(int i=0;i Date: Sat, 26 Oct 2024 00:23:09 -0400 Subject: [PATCH 03/83] Implement function to calculate maximum of IMFP over the grid (UNTESTED) --- Source/Evolve.H | 2 + Source/Evolve.cpp | 136 ++++++++++++++++++++++++++++++++++++++++++++++ Source/main.cpp | 7 +++ 3 files changed, 145 insertions(+) diff --git a/Source/Evolve.H b/Source/Evolve.H index 61e5ada..a797c74 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,6 +26,8 @@ namespace GIdx void Initialize(); } +amrex::Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, MultiFab& mf_IMFP, const TestParams* parms); + amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 0eeff27..c048a9f 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -26,6 +26,142 @@ namespace GIdx } } + +template static inline AMREX_GPU_DEVICE T max3(T a, T b, T c) { + using std::max; + return max(max(a, b), c); +} + +template static inline AMREX_GPU_DEVICE T max4(T a, T b, T c, T d) { + using std::max; + return max(max(a, b), max(c, d)); +} + +template static inline AMREX_GPU_DEVICE T max6(T a, T b, T c, T d, T e, T f) { + using std::max; + return max(max3(a, b, c), max3(d, e, f)); +} + + +Real compute_max_IMFP_from_mf (MultiFab const& mf_IMFP) +{ + auto const& ma = mf_IMFP.const_arrays(); + return ParReduce(TypeList{}, TypeList{}, + mf_IMFP, IntVect(0), // zero ghost cells + [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) + noexcept -> GpuTuple + { + Array4 const& a = ma[box_no]; + return { a(i,j,k) }; + }); +} + + +Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, MultiFab& mf_IMFP, const TestParams* parms){ + //Create NuLib table object + using namespace nulib_private; + NuLib_tabulated NuLib_tabulated_obj(alltables_nulib, logrho_nulib, logtemp_nulib, + yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib); + + int start_comp = GIdx::rho; + int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye + MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); + + for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ + const amrex::Box& bx = mfi.validbox(); + const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); + const amrex::Array4& mf_IMFP_array = mf_IMFP.array(mfi); + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ + + //Get the values from the input arrays + const Real rho = mf_array(i, j, k, GIdx::rho - start_comp); // g/ccm + const Real temperature = mf_array(i, j, k, GIdx::T - start_comp); //erg + const Real Ye = mf_array(i, j, k, GIdx::Ye - start_comp); + + double max_IMFP_abs_local = -10000; //Some random garbage value + + if(parms->IMFP_method==2){ + Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix for nucleon absortion: diag( k_e , k_u , k_t ) + Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for nucleon absortion: diag( kbar_e , kbar_u , kbar_t ) + + //--------------------- Values from NuLib table --------------------------- + int keyerr, anyerr; + double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); + int idx_group = NULIBVAR(idx_group); + //FIXME: specify neutrino energy using the following: + // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. + + //idx_species = {0 for electron neutrino, 1 for electron antineutrino and 2 for all other heavier ones} + //electron neutrino: [0, 0] + int idx_species = 0; + double absorption_opacity, scattering_opacity; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[e] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[e] interpolated = %17.6g\n", scattering_opacity); + #endif + + IMFP_abs[0][0] = absorption_opacity; + + //electron antineutrino: [1, 0] + idx_species = 1; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[a] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[a] interpolated = %17.6g\n", scattering_opacity); + #endif + + IMFP_absbar[0][0] = absorption_opacity; + + //heavier ones: muon neutrino[0,1], muon antineutruino[1,1], tau neutrino[0,2], tau antineutrino[1,2] + idx_species = 2; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[x] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[x] interpolated = %17.6g\n", scattering_opacity); + #endif + + for (int i=1; ineutrino or 1->antineutrino + IMFP_abs[i][i] = absorption_opacity ; // ... fix it ... + IMFP_absbar[i][i] = absorption_opacity ; // ... fix it ... + } + + //Calculate max of all IMFP_abs and IMFP_absbar. + if (NUM_FLAVORS == 2) { + max_IMFP_abs_local = max4(IMFP_abs[0][0], IMFP_abs[1][1], + IMFP_absbar[0][0], IMFP_absbar[1][1]); + } else if (NUM_FLAVORS == 3) { + max_IMFP_abs_local = max6(IMFP_abs[0][0], IMFP_abs[1][1], IMFP_abs[2][2], + IMFP_absbar[0][0], IMFP_absbar[1][1], IMFP_absbar[2][2]); + } + //----------------------------------------------------------------------- + } else { + + assert(0); //Only works for IMFP_method=2 + } + + mf_IMFP_array(i, j, k) = max_IMFP_abs_local; + + }); + } + + //Calculate the reduction of mf_IMFP to calculate the max IMFP value. + //FIXME: FIXME: Do we need additional reduction over MPI ranks? + Real max_IMFP = compute_max_IMFP_from_mf(mf_IMFP); + + return max_IMFP; +} + + Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const TestParams* parms) { AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); diff --git a/Source/main.cpp b/Source/main.cpp index e5ca0a9..899ae08 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -285,6 +285,13 @@ void evolve_flavor(const TestParams* parms) // Get a starting timestep const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); + //------------------------------------------------------------------------------ + //FIXME: FIXME: Define a local multifab inside compute_max_IMFP function itself. Get rid of this here. + MultiFab mf_IMFP(ba, dm, 1, ngrow); + + Real max_IMFP = compute_max_IMFP(geom, state, mf_IMFP, parms); + //------------------------------------------------------------------------------ + // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; From 2c67c2e507e8c17217b35ea123bf330dc87dc813 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 29 Oct 2024 23:00:04 -0400 Subject: [PATCH 04/83] Move multifab for IMFB inside compute_max_IMFP function, correct temperature units --- Source/Evolve.H | 3 ++- Source/Evolve.cpp | 9 +++++++-- Source/main.cpp | 6 ++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index a797c74..f3eceab 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,7 +26,8 @@ namespace GIdx void Initialize(); } -amrex::Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, MultiFab& mf_IMFP, const TestParams* parms); +amrex::Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms, + const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, const IntVect& ngrow); amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index c048a9f..50d98d5 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -57,7 +57,8 @@ Real compute_max_IMFP_from_mf (MultiFab const& mf_IMFP) } -Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, MultiFab& mf_IMFP, const TestParams* parms){ +Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms, + const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, const IntVect& ngrow){ //Create NuLib table object using namespace nulib_private; NuLib_tabulated NuLib_tabulated_obj(alltables_nulib, logrho_nulib, logtemp_nulib, @@ -67,6 +68,8 @@ Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, MultiFab& mf_ int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); + MultiFab mf_IMFP(ba, dm, 1, ngrow); //ncomp=1 + for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.validbox(); const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); @@ -76,9 +79,11 @@ Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, MultiFab& mf_ //Get the values from the input arrays const Real rho = mf_array(i, j, k, GIdx::rho - start_comp); // g/ccm - const Real temperature = mf_array(i, j, k, GIdx::T - start_comp); //erg + const Real T_grid = mf_array(i, j, k, GIdx::T - start_comp); //erg const Real Ye = mf_array(i, j, k, GIdx::Ye - start_comp); + const Real temperature = T_grid/ (1e6*CGSUnitsConst::eV); //convert temperature from erg to MeV. + double max_IMFP_abs_local = -10000; //Some random garbage value if(parms->IMFP_method==2){ diff --git a/Source/main.cpp b/Source/main.cpp index 899ae08..d90e9d4 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -286,10 +286,8 @@ void evolve_flavor(const TestParams* parms) const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); //------------------------------------------------------------------------------ - //FIXME: FIXME: Define a local multifab inside compute_max_IMFP function itself. Get rid of this here. - MultiFab mf_IMFP(ba, dm, 1, ngrow); - - Real max_IMFP = compute_max_IMFP(geom, state, mf_IMFP, parms); + Real max_IMFP = compute_max_IMFP(geom, state, parms, ba, dm, ngrow); + //printf("max_IMFP = %g\n", max_IMFP); //TODO: Remove comment //------------------------------------------------------------------------------ // Do all the science! From 795615cd0e47b3761bd072d3faff7502ec99b003 Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 29 Oct 2024 23:13:27 -0400 Subject: [PATCH 05/83] do not input ba, dm and ngrow inside compute_max_IMFP function --- Source/Evolve.H | 3 +-- Source/Evolve.cpp | 6 +++--- Source/main.cpp | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index f3eceab..01a6a8f 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,8 +26,7 @@ namespace GIdx void Initialize(); } -amrex::Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms, - const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, const IntVect& ngrow); +amrex::Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms); amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 50d98d5..9e61aad 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -57,8 +57,7 @@ Real compute_max_IMFP_from_mf (MultiFab const& mf_IMFP) } -Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms, - const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, const IntVect& ngrow){ +Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ //Create NuLib table object using namespace nulib_private; NuLib_tabulated NuLib_tabulated_obj(alltables_nulib, logrho_nulib, logtemp_nulib, @@ -68,7 +67,8 @@ Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestPar int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); - MultiFab mf_IMFP(ba, dm, 1, ngrow); //ncomp=1 + const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP + MultiFab mf_IMFP(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.validbox(); diff --git a/Source/main.cpp b/Source/main.cpp index d90e9d4..88da0f9 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -286,8 +286,8 @@ void evolve_flavor(const TestParams* parms) const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); //------------------------------------------------------------------------------ - Real max_IMFP = compute_max_IMFP(geom, state, parms, ba, dm, ngrow); - //printf("max_IMFP = %g\n", max_IMFP); //TODO: Remove comment + Real max_IMFP = compute_max_IMFP(geom, state, parms); + printf("max_IMFP = %g\n", max_IMFP); //TODO: Remove comment //------------------------------------------------------------------------------ // Do all the science! From a9d22c47115931f1bd667292dfab212d223db94c Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Tue, 29 Oct 2024 23:21:08 -0400 Subject: [PATCH 06/83] calculate max_IMFP_abs inside function compute_dt Currently compute_dt runs every iteration. We may need to run compute_dt just once initially during production, since (rho,T,Ye) is static. --- Source/Evolve.cpp | 7 +++++++ Source/main.cpp | 5 ----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 9e61aad..46e5383 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -228,6 +228,13 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr } // Calculate dt_flavor_absorption dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; + + } else if (parms->IMFP_method == 2) { + double max_IMFP_abs = compute_max_IMFP(geom, state, parms); + printf("max_IMFP_abs = %g\n", max_IMFP_abs); //TODO: Remove comment + // Calculate dt_flavor_absorption + dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; + } // pick the appropriate timestep diff --git a/Source/main.cpp b/Source/main.cpp index 88da0f9..e5ca0a9 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -285,11 +285,6 @@ void evolve_flavor(const TestParams* parms) // Get a starting timestep const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); - //------------------------------------------------------------------------------ - Real max_IMFP = compute_max_IMFP(geom, state, parms); - printf("max_IMFP = %g\n", max_IMFP); //TODO: Remove comment - //------------------------------------------------------------------------------ - // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; From 66eeb3c4f033ef2d3b178502b7317718db3beb3a Mon Sep 17 00:00:00 2001 From: shankar-1729 Date: Thu, 31 Oct 2024 15:57:31 -0400 Subject: [PATCH 07/83] Remove dt calculation from compute_dt function --- Source/Evolve.cpp | 4 ++-- Source/main.cpp | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 46e5383..c630705 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -229,13 +229,13 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr // Calculate dt_flavor_absorption dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; - } else if (parms->IMFP_method == 2) { + } /*else if (parms->IMFP_method == 2) { double max_IMFP_abs = compute_max_IMFP(geom, state, parms); printf("max_IMFP_abs = %g\n", max_IMFP_abs); //TODO: Remove comment // Calculate dt_flavor_absorption dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; - } + }*/ // pick the appropriate timestep dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); diff --git a/Source/main.cpp b/Source/main.cpp index e5ca0a9..33546bb 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -285,6 +285,12 @@ void evolve_flavor(const TestParams* parms) // Get a starting timestep const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); + //------------------------------------------------------------------------------ + Real max_IMFP = compute_max_IMFP(geom, state, parms); + printf("max_IMFP = %g\n", max_IMFP); //TODO: Remove comment + //------------------------------------------------------------------------------ + + // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; From 3480a61ae67a69caac1e018c915a86f2505544b9 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 4 Nov 2024 11:24:06 -0500 Subject: [PATCH 08/83] Adding basics comments and documentation to new functions --- Source/Evolve.cpp | 59 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index c630705..cc4bf70 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -26,23 +26,78 @@ namespace GIdx } } - +/** + * @brief Computes the maximum of three values. + * + * This function takes three arguments of the same type and returns the maximum value among them. + * It uses the standard library's `max` function to perform the comparison. + * + * @tparam T The type of the input values. Must support comparison via the `max` function. + * @param a The first value to compare. + * @param b The second value to compare. + * @param c The third value to compare. + * @return The maximum value among the three input values. + */ template static inline AMREX_GPU_DEVICE T max3(T a, T b, T c) { using std::max; return max(max(a, b), c); } +/** + * @brief Computes the maximum of four values. + * + * This function takes four values of the same type and returns the maximum + * value among them. It uses the standard library's `max` function to compare + * the values. + * + * @tparam T The type of the values to be compared. This type must support + * comparison using the `max` function. + * @param a The first value to compare. + * @param b The second value to compare. + * @param c The third value to compare. + * @param d The fourth value to compare. + * @return The maximum value among the four input values. + */ template static inline AMREX_GPU_DEVICE T max4(T a, T b, T c, T d) { using std::max; return max(max(a, b), max(c, d)); } +/** + * @brief Computes the maximum value among six given values. + * + * This function takes six input values of type T and returns the maximum + * value among them. It utilizes the `max3` function to find the maximum + * of three values and then compares the results of two such calls to + * determine the overall maximum. + * + * @tparam T The type of the input values. It should support comparison + * operations. + * @param a The first value to compare. + * @param b The second value to compare. + * @param c The third value to compare. + * @param d The fourth value to compare. + * @param e The fifth value to compare. + * @param f The sixth value to compare. + * @return The maximum value among the six input values. + */ template static inline AMREX_GPU_DEVICE T max6(T a, T b, T c, T d, T e, T f) { using std::max; return max(max3(a, b, c), max3(d, e, f)); } - +/** + * @brief Computes the maximum Inverse Mean Free Path (IMFP) from a given MultiFab. + * + * This function performs a parallel reduction to find the maximum value of the + * Inverse Mean Free Path (IMFP) stored in the provided MultiFab. It uses the + * ParReduce function with a maximum reduction operation. + * + * @param mf_IMFP The MultiFab containing the IMFP values. It is assumed that + * the MultiFab is properly initialized and contains valid data. + * + * @return The maximum IMFP value found in the MultiFab. + */ Real compute_max_IMFP_from_mf (MultiFab const& mf_IMFP) { auto const& ma = mf_IMFP.const_arrays(); From 8458eae166fb7afd52b882141d5df8ddeb81520c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 4 Nov 2024 13:01:37 -0500 Subject: [PATCH 09/83] Change the position of the calculation of the maximum IMFP --- Source/main.cpp | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index 33546bb..0464f15 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -138,7 +138,11 @@ void evolve_flavor(const TestParams* parms) amrex::Print() << "Reading NuLib table... " << std::endl; ReadNuLibTable(parms->nulib_table_name); } - + + // Compute the maximum Inverse Mean Free Path (IMFP) in the domain + Real max_IMFP = compute_max_IMFP(geom, state, parms); + printf("max_IMFP = %g\n", max_IMFP); //TODO: Remove comment + // Initialize particles on the domain amrex::Print() << "Initializing particles... " << std::endl; @@ -251,15 +255,15 @@ void evolve_flavor(const TestParams* parms) const int step = integrator.get_step_number(); const Real time = integrator.get_time(); - printf("Writing reduced data to file... \n"); - rd.WriteReducedData0D(geom, state, neutrinos, time, step+1); - printf("Done. \n"); + printf("Writing reduced data to file... \n"); + rd.WriteReducedData0D(geom, state, neutrinos, time, step+1); + printf("Done. \n"); run_fom += neutrinos.TotalNumberOfParticles(); // Write the Mesh Data to Plotfile if required - bool write_plotfile = parms->write_plot_every > 0 && (step+1) % parms->write_plot_every == 0; - bool write_plot_particles = parms->write_plot_particles_every > 0 && (step+1) % parms->write_plot_particles_every == 0; + bool write_plotfile = parms->write_plot_every > 0 && (step+1) % parms->write_plot_every == 0; + bool write_plot_particles = parms->write_plot_particles_every > 0 && (step+1) % parms->write_plot_particles_every == 0; if (write_plotfile || write_plot_particles) { // Only include the Particle Data if write_plot_particles_every is satisfied int write_plot_particles = parms->write_plot_particles_every > 0 && @@ -285,12 +289,6 @@ void evolve_flavor(const TestParams* parms) // Get a starting timestep const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); - //------------------------------------------------------------------------------ - Real max_IMFP = compute_max_IMFP(geom, state, parms); - printf("max_IMFP = %g\n", max_IMFP); //TODO: Remove comment - //------------------------------------------------------------------------------ - - // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; From 792c8fdf308249929c375bfc992a3ee3cd060ed2 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 4 Nov 2024 13:12:16 -0500 Subject: [PATCH 10/83] Compute maximum of IMFP when IMFP method is 1 too --- Source/Evolve.cpp | 221 +++++++++++++++++++++++++--------------------- 1 file changed, 121 insertions(+), 100 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index cc4bf70..ac90e67 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -113,112 +113,133 @@ Real compute_max_IMFP_from_mf (MultiFab const& mf_IMFP) Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ - //Create NuLib table object - using namespace nulib_private; - NuLib_tabulated NuLib_tabulated_obj(alltables_nulib, logrho_nulib, logtemp_nulib, - yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib); - int start_comp = GIdx::rho; - int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye - MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); - - const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP - MultiFab mf_IMFP(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 - - for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ - const amrex::Box& bx = mfi.validbox(); - const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); - const amrex::Array4& mf_IMFP_array = mf_IMFP.array(mfi); - - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ - - //Get the values from the input arrays - const Real rho = mf_array(i, j, k, GIdx::rho - start_comp); // g/ccm - const Real T_grid = mf_array(i, j, k, GIdx::T - start_comp); //erg - const Real Ye = mf_array(i, j, k, GIdx::Ye - start_comp); - - const Real temperature = T_grid/ (1e6*CGSUnitsConst::eV); //convert temperature from erg to MeV. - - double max_IMFP_abs_local = -10000; //Some random garbage value - - if(parms->IMFP_method==2){ - Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix for nucleon absortion: diag( k_e , k_u , k_t ) - Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for nucleon absortion: diag( kbar_e , kbar_u , kbar_t ) - - //--------------------- Values from NuLib table --------------------------- - int keyerr, anyerr; - double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); - int idx_group = NULIBVAR(idx_group); - //FIXME: specify neutrino energy using the following: - // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. - - //idx_species = {0 for electron neutrino, 1 for electron antineutrino and 2 for all other heavier ones} - //electron neutrino: [0, 0] - int idx_species = 0; - double absorption_opacity, scattering_opacity; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - if (anyerr) assert(0); - #ifdef DEBUG_INTERPOLATION_TABLES - printf("(Evolve.cpp) absorption_opacity[e] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[e] interpolated = %17.6g\n", scattering_opacity); - #endif - - IMFP_abs[0][0] = absorption_opacity; - - //electron antineutrino: [1, 0] - idx_species = 1; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - if (anyerr) assert(0); - - #ifdef DEBUG_INTERPOLATION_TABLES - printf("(Evolve.cpp) absorption_opacity[a] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[a] interpolated = %17.6g\n", scattering_opacity); - #endif - - IMFP_absbar[0][0] = absorption_opacity; - - //heavier ones: muon neutrino[0,1], muon antineutruino[1,1], tau neutrino[0,2], tau antineutrino[1,2] - idx_species = 2; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - if (anyerr) assert(0); - - #ifdef DEBUG_INTERPOLATION_TABLES - printf("(Evolve.cpp) absorption_opacity[x] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[x] interpolated = %17.6g\n", scattering_opacity); - #endif - - for (int i=1; ineutrino or 1->antineutrino - IMFP_abs[i][i] = absorption_opacity ; // ... fix it ... - IMFP_absbar[i][i] = absorption_opacity ; // ... fix it ... - } + // If IMFP_method is 1, use the IMFPs from the input file and find the maximum absorption IMFP + if (parms->IMFP_method == 1) { + + // Use the IMFPs from the input file and find the maximum absorption IMFP + double max_IMFP_abs = std::numeric_limits::lowest(); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < NUM_FLAVORS; ++j) { + max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); + } + } + return max_IMFP_abs; - //Calculate max of all IMFP_abs and IMFP_absbar. - if (NUM_FLAVORS == 2) { - max_IMFP_abs_local = max4(IMFP_abs[0][0], IMFP_abs[1][1], - IMFP_absbar[0][0], IMFP_absbar[1][1]); - } else if (NUM_FLAVORS == 3) { - max_IMFP_abs_local = max6(IMFP_abs[0][0], IMFP_abs[1][1], IMFP_abs[2][2], - IMFP_absbar[0][0], IMFP_absbar[1][1], IMFP_absbar[2][2]); + // If IMFP_method is 2, use the NuLib table to find the maximum absorption IMFP + }else if (parms->IMFP_method == 2) + { + //Create NuLib table object + using namespace nulib_private; + NuLib_tabulated NuLib_tabulated_obj(alltables_nulib, logrho_nulib, logtemp_nulib, + yes_nulib, helperVarsReal_nulib, helperVarsInt_nulib); + + int start_comp = GIdx::rho; + int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye + MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); + + const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP + MultiFab mf_IMFP(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 + + for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ + const amrex::Box& bx = mfi.validbox(); + const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); + const amrex::Array4& mf_IMFP_array = mf_IMFP.array(mfi); + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ + + //Get the values from the input arrays + const Real rho = mf_array(i, j, k, GIdx::rho - start_comp); // g/ccm + const Real T_grid = mf_array(i, j, k, GIdx::T - start_comp); //erg + const Real Ye = mf_array(i, j, k, GIdx::Ye - start_comp); + + const Real temperature = T_grid/ (1e6*CGSUnitsConst::eV); //convert temperature from erg to MeV. + + double max_IMFP_abs_local = -10000; //Some random garbage value + + if(parms->IMFP_method==2){ + Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix for nucleon absortion: diag( k_e , k_u , k_t ) + Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for nucleon absortion: diag( kbar_e , kbar_u , kbar_t ) + + //--------------------- Values from NuLib table --------------------------- + int keyerr, anyerr; + double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); + int idx_group = NULIBVAR(idx_group); + //FIXME: specify neutrino energy using the following: + // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. + + //idx_species = {0 for electron neutrino, 1 for electron antineutrino and 2 for all other heavier ones} + //electron neutrino: [0, 0] + int idx_species = 0; + double absorption_opacity, scattering_opacity; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[e] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[e] interpolated = %17.6g\n", scattering_opacity); + #endif + + IMFP_abs[0][0] = absorption_opacity; + + //electron antineutrino: [1, 0] + idx_species = 1; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[a] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[a] interpolated = %17.6g\n", scattering_opacity); + #endif + + IMFP_absbar[0][0] = absorption_opacity; + + //heavier ones: muon neutrino[0,1], muon antineutruino[1,1], tau neutrino[0,2], tau antineutrino[1,2] + idx_species = 2; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[x] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[x] interpolated = %17.6g\n", scattering_opacity); + #endif + + for (int i=1; ineutrino or 1->antineutrino + IMFP_abs[i][i] = absorption_opacity ; // ... fix it ... + IMFP_absbar[i][i] = absorption_opacity ; // ... fix it ... + } + + //Calculate max of all IMFP_abs and IMFP_absbar. + if (NUM_FLAVORS == 2) { + max_IMFP_abs_local = max4(IMFP_abs[0][0], IMFP_abs[1][1], + IMFP_absbar[0][0], IMFP_absbar[1][1]); + } else if (NUM_FLAVORS == 3) { + max_IMFP_abs_local = max6(IMFP_abs[0][0], IMFP_abs[1][1], IMFP_abs[2][2], + IMFP_absbar[0][0], IMFP_absbar[1][1], IMFP_absbar[2][2]); + } + //----------------------------------------------------------------------- + } else { + + assert(0); //Only works for IMFP_method=2 } - //----------------------------------------------------------------------- - } else { - assert(0); //Only works for IMFP_method=2 - } - - mf_IMFP_array(i, j, k) = max_IMFP_abs_local; - - }); - } + mf_IMFP_array(i, j, k) = max_IMFP_abs_local; + + }); + } - //Calculate the reduction of mf_IMFP to calculate the max IMFP value. - //FIXME: FIXME: Do we need additional reduction over MPI ranks? - Real max_IMFP = compute_max_IMFP_from_mf(mf_IMFP); + //Calculate the reduction of mf_IMFP to calculate the max IMFP value. + //FIXME: FIXME: Do we need additional reduction over MPI ranks? + Real max_IMFP = compute_max_IMFP_from_mf(mf_IMFP); - return max_IMFP; + return max_IMFP; + }else + { + amrex::Error("Invalid IMFP method. Please check the input file."); + return -1; + } } From 4b43fff75c46eeca875d104ce76fda56aa38cda5 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 4 Nov 2024 13:28:54 -0500 Subject: [PATCH 11/83] Compute the maximum IMFP and pass it as a parameter to the compute_dt function. --- Source/Evolve.H | 2 +- Source/Evolve.cpp | 13 +++---------- Source/main.cpp | 11 +++++++---- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 01a6a8f..425c1ec 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -28,7 +28,7 @@ namespace GIdx amrex::Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms); -amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); +amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms, Real maximum_IMFP_abs); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index ac90e67..89fe64b 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -243,7 +243,7 @@ Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestPar } -Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const TestParams* parms) +Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const TestParams* parms, Real maximum_IMFP_abs) { AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); @@ -294,16 +294,9 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; } - if (parms->IMFP_method == 1) { - // Use the IMFPs from the input file and find the maximum absorption IMFP - double max_IMFP_abs = std::numeric_limits::lowest(); // Initialize max to lowest possible value - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < NUM_FLAVORS; ++j) { - max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); - } - } + if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { // Calculate dt_flavor_absorption - dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; + dt_flavor_absorption = (1 / (PhysConst::c * maximum_IMFP_abs)) * parms->collision_cfl_factor; } /*else if (parms->IMFP_method == 2) { double max_IMFP_abs = compute_max_IMFP(geom, state, parms); diff --git a/Source/main.cpp b/Source/main.cpp index 0464f15..faf7735 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -140,8 +140,11 @@ void evolve_flavor(const TestParams* parms) } // Compute the maximum Inverse Mean Free Path (IMFP) in the domain - Real max_IMFP = compute_max_IMFP(geom, state, parms); - printf("max_IMFP = %g\n", max_IMFP); //TODO: Remove comment + Real max_IMFP_absortion = 0.0; + if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { + max_IMFP_absortion = compute_max_IMFP(geom, state, parms); + printf("max_IMFP_absortion = %g\n", max_IMFP_absortion); //TODO: Remove comment + } // Initialize particles on the domain amrex::Print() << "Initializing particles... " << std::endl; @@ -276,7 +279,7 @@ void evolve_flavor(const TestParams* parms) // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. printf("Setting next timestep... \n"); - const Real dt = compute_dt(geom, state, neutrinos, parms); + const Real dt = compute_dt(geom, state, neutrinos, parms, max_IMFP_absortion); integrator.set_timestep(dt); //printf("current_dt = %g, dt = %g \n", current_dt, dt); printf("Done. \n"); @@ -287,7 +290,7 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms); + const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms, max_IMFP_absortion); // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; From db60725d9e580233d4f11023cee4148e0cdc2acd Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 4 Nov 2024 13:46:37 -0500 Subject: [PATCH 12/83] Cleaning up the compute_dt function and adding documentation. --- Source/Evolve.cpp | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 89fe64b..6d77aaf 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -242,8 +242,26 @@ Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestPar } } - -Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& /* neutrinos */, const TestParams* parms, Real maximum_IMFP_abs) +/** + * @brief Computes the time step for the simulation based on various CFL factors. + * + * This function calculates the time step for the simulation by considering the + * CFL factors for translation, flavor, and collision. It uses the maximum IMFP + * value to determine the collision time step and combines it with the translation + * and flavor time steps to find the minimum time step for the simulation. + * + * @param geom The geometry of the simulation domain. + * @param state The MultiFab containing the state variables. + * @param neutrinos The container holding the flavored neutrinos. + * @param maximum_IMFP_abs The maximum inverse mean free path for absorption. + * @return The computed time step for the simulation. + */ +Real compute_dt( + const Geometry& geom, + const MultiFab& state, + const FlavoredNeutrinoContainer&, + const TestParams* parms, + Real maximum_IMFP_abs) { AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); @@ -251,7 +269,7 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr const auto dxi = geom.CellSizeArray(); Real dt_translation = 0.0; if (parms->cfl_factor > 0.0) { - dt_translation = std::min(std::min(dxi[0],dxi[1]), dxi[2]) / PhysConst::c * parms->cfl_factor; + dt_translation = std::min(std::min(dxi[0], dxi[1]), dxi[2]) / PhysConst::c * parms->cfl_factor; } Real dt_flavor = 0.0; @@ -265,7 +283,7 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr using ReduceTuple = typename decltype(reduce_data)::Type; for (MFIter mfi(state); mfi.isValid(); ++mfi) { const Box& bx = mfi.fabbox(); - auto const& fab = state.array(mfi); + auto const& fab = state.array(mfi); reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { @@ -273,7 +291,7 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr #include "generated_files/Evolve.cpp_compute_dt_fill" return {V_adaptive, V_stupid}; }); - } + } // extract the reduced values from the combined reduced data structure auto rv = reduce_data.value(); @@ -298,17 +316,11 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr // Calculate dt_flavor_absorption dt_flavor_absorption = (1 / (PhysConst::c * maximum_IMFP_abs)) * parms->collision_cfl_factor; - } /*else if (parms->IMFP_method == 2) { - double max_IMFP_abs = compute_max_IMFP(geom, state, parms); - printf("max_IMFP_abs = %g\n", max_IMFP_abs); //TODO: Remove comment - // Calculate dt_flavor_absorption - dt_flavor_absorption = (1 / (PhysConst::c * max_IMFP_abs)) * parms->collision_cfl_factor; - - }*/ + } // pick the appropriate timestep dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); - if(parms->max_adaptive_speedup>1) { + if(parms->max_adaptive_speedup > 1) { dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); } } @@ -321,7 +333,7 @@ Real compute_dt(const Geometry& geom, const MultiFab& state, const FlavoredNeutr } else if (dt_flavor != 0.0) { dt = dt_flavor; } else { - amrex::Error("Timestep selection failed, try using both cfl_factor and flavor_cfl_factor"); + amrex::Error("Timestep selection failed, both dt_translation and dt_flavor are zero. Try using both cfl_factor and flavor_cfl_factor."); } return dt; From d02f8657c98decd6be06490209e884c434b04c91 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 4 Nov 2024 16:27:25 -0500 Subject: [PATCH 13/83] Include a generated file to compute the trace of the neutrino number density for mesh quantities --- Scripts/symbolic_hermitians/generate_code.py | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 2e223e9..feaddea 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -526,4 +526,23 @@ def sgn(t,var): code.append(dNdt.code()) code = [line for sublist in code for line in sublist] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill_zeros")) \ No newline at end of file + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill_zeros")) + + #========================# + # Evolve.cpp_compute_trace # + #========================# + + # List that will store the code for setting the derivative of the matrices N and Nbar to zero. + code = [] + + # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) + for t in tails: + + # Define n and nbar matrix + N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Derivative of the neutrino number matrix + # Assign the trace of n and nbar to a variable + trace_N = N.trace() + code.append(["trace_n"+t+" = " + sympy.cxxcode(sympy.simplify(trace_N)) + ";"]) + + code = [line for sublist in code for line in sublist] + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_compute_trace")) \ No newline at end of file From 0959355336679efdf4fbe3042d13f0c2539bd3ee Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 4 Nov 2024 16:58:38 -0500 Subject: [PATCH 14/83] Solving issue in generated file to compute trace of mesh neutrino and antineutrino number densities --- Scripts/symbolic_hermitians/generate_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index feaddea..39a2dcc 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -539,7 +539,7 @@ def sgn(t,var): for t in tails: # Define n and nbar matrix - N = HermitianMatrix(args.N, "p.rdata(PIdx::N{}{}_{}"+t+")") # Derivative of the neutrino number matrix + N = HermitianMatrix(args.N, "mf_array(i\,j\,k\,GIdx::N{}{}_{}"+t+"-start_comp)") # Assign the trace of n and nbar to a variable trace_N = N.trace() code.append(["trace_n"+t+" = " + sympy.cxxcode(sympy.simplify(trace_N)) + ";"]) From b4c008bca390db059960874bf7452ab52e232d24 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 4 Nov 2024 16:59:39 -0500 Subject: [PATCH 15/83] Compute maximum trace of the neutrino and antineutrino from mesh quantities to compute the correct time step --- Source/Evolve.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 6d77aaf..5590874 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -263,6 +263,40 @@ Real compute_dt( const TestParams* parms, Real maximum_IMFP_abs) { + + // ################################################################## + + + // Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ + + int start_comp = GIdx::rho; + int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye + MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); + + const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP + MultiFab multifab_trace_n(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 + + for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ + + const amrex::Box& bx = mfi.validbox(); + const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); + const amrex::Array4& multifab_trace_n_array = multifab_trace_n.array(mfi); + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ + + Real trace_n = 0.0; + Real trace_nbar = 0.0; + #include "generated_files/Evolve.cpp_compute_trace" + multifab_trace_n_array(i, j, k) = max(trace_n, trace_nbar); + + }); + } + + Real max_trace = compute_max_IMFP_from_mf(multifab_trace_n); + printf("max_trace = %g\n", max_trace); + + // ################################################################## + AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); // translation part of timestep limit From 0399808336af43f77d13e8bccf57d01d06632139 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 09:27:31 -0500 Subject: [PATCH 16/83] Create a function that computes the minimum value in a MultiFab in parallel. This function will be used to compute the minimum trace of mesh quantities, and based on that, it will compute the optimum time step --- Source/Evolve.cpp | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 5590874..1fa3fa0 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -111,6 +111,30 @@ Real compute_max_IMFP_from_mf (MultiFab const& mf_IMFP) }); } +/** + * @brief Computes the minimum value in a MultiFab. + * + * This function performs a parallel reduction to find the minimum value + * stored in the provided MultiFab. It uses the + * ParReduce function with a minimum reduction operation. + * + * @param multifab The MultiFab containing the values. It is assumed that + * the MultiFab is properly initialized and contains valid data. + * + * @return The minimum value found in the MultiFab. + */ +Real compute_min_of_multifab (MultiFab const& multifab) +{ + auto const& ma = multifab.const_arrays(); + return ParReduce(TypeList{}, TypeList{}, + multifab, IntVect(0), // zero ghost cells + [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) + noexcept -> GpuTuple + { + Array4 const& a = ma[box_no]; + return { a(i,j,k) }; + }); +} Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ @@ -266,9 +290,6 @@ Real compute_dt( // ################################################################## - - // Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ - int start_comp = GIdx::rho; int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); @@ -292,8 +313,8 @@ Real compute_dt( }); } - Real max_trace = compute_max_IMFP_from_mf(multifab_trace_n); - printf("max_trace = %g\n", max_trace); + Real min_trace = compute_min_of_multifab(multifab_trace_n); + printf("min_trace = %g\n", min_trace); // ################################################################## @@ -344,12 +365,14 @@ Real compute_dt( if (parms->attenuation_hamiltonians != 0) { dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + dt_flavor_adaptive *= min_trace; + dt_flavor_stupid *= min_trace; } if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { // Calculate dt_flavor_absorption dt_flavor_absorption = (1 / (PhysConst::c * maximum_IMFP_abs)) * parms->collision_cfl_factor; - + dt_flavor_absorption *= min_trace; } // pick the appropriate timestep From be783b96e08730f845bf9a2c1dbb35fe0a3c9ecf Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 13:10:27 -0500 Subject: [PATCH 17/83] Set some if statements to handle simulations with empty periodic boundary conditions and black holes. --- Source/Evolve.cpp | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 1fa3fa0..df9a938 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -307,6 +307,37 @@ Real compute_dt( Real trace_n = 0.0; Real trace_nbar = 0.0; + + if (parms->do_periodic_empty_bc == 1) { + + if (i == 0 || i == geom.Domain().length(0) - 1 || + j == 0 || j == geom.Domain().length(1) - 1 || + k == 0 || k == geom.Domain().length(2) - 1) { + + multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); + return; + + } + } + + if (parms->do_black_hole == 1) { + + double cell_size_x = parms->Lx / parms->ncell[0]; + double cell_size_y = parms->Ly / parms->ncell[1]; + double cell_size_z = parms->Lz / parms->ncell[2]; + + double x_cell_center = (i + 0.5) * cell_size_x; + double y_cell_center = (j + 0.5) * cell_size_y; + double z_cell_center = (k + 0.5) * cell_size_z; + + distance_from_bh = sqrt(pow(x_cell_center - parms->bh_x, 2) + pow(y_cell_center - parms->bh_y, 2) + pow(z_cell_center - parms->bh_z, 2)); + + if (distance_from_bh < parms->bh_radius) { + multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); + return; + } + } + #include "generated_files/Evolve.cpp_compute_trace" multifab_trace_n_array(i, j, k) = max(trace_n, trace_nbar); @@ -344,6 +375,9 @@ Real compute_dt( { Real V_adaptive=0, V_adaptive2=0, V_stupid=0; #include "generated_files/Evolve.cpp_compute_dt_fill" + #include "generated_files/Evolve.cpp_compute_trace" + V_adaptive *= max(trace_n, trace_nbar) + V_stupid *= max(trace_n, trace_nbar) return {V_adaptive, V_stupid}; }); } @@ -365,8 +399,6 @@ Real compute_dt( if (parms->attenuation_hamiltonians != 0) { dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; - dt_flavor_adaptive *= min_trace; - dt_flavor_stupid *= min_trace; } if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { From d8e9f1e38a60d92cd8f3c69dc62057aa04a38364 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 13:46:36 -0500 Subject: [PATCH 18/83] Generate a new file to compute the trace on mesh quantities --- Scripts/symbolic_hermitians/generate_code.py | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index 39a2dcc..d9232bf 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -545,4 +545,23 @@ def sgn(t,var): code.append(["trace_n"+t+" = " + sympy.cxxcode(sympy.simplify(trace_N)) + ";"]) code = [line for sublist in code for line in sublist] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_compute_trace")) \ No newline at end of file + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_compute_trace")) + + #========================# + # Evolve.cpp_compute_trace_2 # + #========================# + + # List that will store the code for setting the derivative of the matrices N and Nbar to zero. + code = [] + + # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) + for t in tails: + + # Define n and nbar matrix + N = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{}"+t+")") + # Assign the trace of n and nbar to a variable + trace_N = N.trace() + code.append(["trace_n"+t+" = " + sympy.cxxcode(sympy.simplify(trace_N)) + ";"]) + + code = [line for sublist in code for line in sublist] + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_compute_trace_2")) \ No newline at end of file From c5c70ed033152919bd7663a22304a4a9247f7f3c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 13:47:26 -0500 Subject: [PATCH 19/83] Solve syntaxis mistakes and compute the minumin time step rather than the minimum potential --- Source/Evolve.cpp | 77 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index df9a938..ac30445 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -310,9 +310,9 @@ Real compute_dt( if (parms->do_periodic_empty_bc == 1) { - if (i == 0 || i == geom.Domain().length(0) - 1 || - j == 0 || j == geom.Domain().length(1) - 1 || - k == 0 || k == geom.Domain().length(2) - 1) { + if (i == 0 || i == parms->ncell[0] - 1 || + j == 0 || j == parms->ncell[1] - 1 || + k == 0 || k == parms->ncell[2] - 1) { multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); return; @@ -320,7 +320,7 @@ Real compute_dt( } } - if (parms->do_black_hole == 1) { + if (parms->do_blackhole == 1) { double cell_size_x = parms->Lx / parms->ncell[0]; double cell_size_y = parms->Ly / parms->ncell[1]; @@ -330,7 +330,7 @@ Real compute_dt( double y_cell_center = (j + 0.5) * cell_size_y; double z_cell_center = (k + 0.5) * cell_size_z; - distance_from_bh = sqrt(pow(x_cell_center - parms->bh_x, 2) + pow(y_cell_center - parms->bh_y, 2) + pow(z_cell_center - parms->bh_z, 2)); + Real distance_from_bh = sqrt(pow(x_cell_center - parms->bh_center_x, 2) + pow(y_cell_center - parms->bh_center_y, 2) + pow(z_cell_center - parms->bh_center_z, 2)); if (distance_from_bh < parms->bh_radius) { multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); @@ -364,32 +364,77 @@ Real compute_dt( // the potential from matter and neutrinos // compute "effective" potential (ergs) that produces characteristic timescale // when multiplied by hbar - ReduceOps reduce_op; + // ReduceOps reduce_op; + // ReduceData reduce_data(reduce_op); + // using ReduceTuple = typename decltype(reduce_data)::Type; + // for (MFIter mfi(state); mfi.isValid(); ++mfi) { + // const Box& bx = mfi.fabbox(); + // auto const& fab = state.array(mfi); + // reduce_op.eval(bx, reduce_data, + // [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple + // { + // Real V_adaptive=0, V_adaptive2=0, V_stupid=0; + // #include "generated_files/Evolve.cpp_compute_dt_fill" + + // Real trace_n = 0.0, trace_nbar = 0.0; + // #include "generated_files/Evolve.cpp_compute_trace" + + // V_adaptive *= max(trace_n, trace_nbar); + // V_stupid *= max(trace_n, trace_nbar); + // return {V_adaptive, V_stupid}; + // }); + // } + + // // extract the reduced values from the combined reduced data structure + // auto rv = reduce_data.value(); + // Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; + // Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; + + // // reduce across MPI ranks + // ParallelDescriptor::ReduceRealMax(Vmax_adaptive); + // ParallelDescriptor::ReduceRealMax(Vmax_stupid ); + + // ################################################################## + + ReduceOps reduce_op; ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; for (MFIter mfi(state); mfi.isValid(); ++mfi) { const Box& bx = mfi.fabbox(); auto const& fab = state.array(mfi); + Real V_vac_max = FlavoredNeutrinoContainer::Vvac_max; reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { Real V_adaptive=0, V_adaptive2=0, V_stupid=0; #include "generated_files/Evolve.cpp_compute_dt_fill" - #include "generated_files/Evolve.cpp_compute_trace" - V_adaptive *= max(trace_n, trace_nbar) - V_stupid *= max(trace_n, trace_nbar) - return {V_adaptive, V_stupid}; + + V_adaptive += V_vac_max; + V_stupid += V_vac_max; + + Real trace_n = 0.0, trace_nbar = 0.0; + #include "generated_files/Evolve.cpp_compute_trace_2" + + Real min_trace = min(trace_n, trace_nbar); + + Real dt_adaptive = min_trace / V_adaptive; + Real dt_stupid = min_trace / V_stupid; + + return {dt_adaptive, dt_stupid}; }); } // extract the reduced values from the combined reduced data structure auto rv = reduce_data.value(); - Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; - Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; + Real min_dt_adaptive = amrex::get<0>(rv); + Real min_dt_stupid = amrex::get<1>(rv); // reduce across MPI ranks - ParallelDescriptor::ReduceRealMax(Vmax_adaptive); - ParallelDescriptor::ReduceRealMax(Vmax_stupid ); + ParallelDescriptor::ReduceRealMin(min_dt_adaptive); + ParallelDescriptor::ReduceRealMin(min_dt_stupid ); + + // ################################################################## + // define the dt associated with each method Real dt_flavor_adaptive = std::numeric_limits::max(); @@ -397,8 +442,8 @@ Real compute_dt( Real dt_flavor_absorption = std::numeric_limits::max(); // Initialize with infinity if (parms->attenuation_hamiltonians != 0) { - dt_flavor_adaptive = PhysConst::hbar / Vmax_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; - dt_flavor_stupid = PhysConst::hbar / Vmax_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + dt_flavor_adaptive = PhysConst::hbar * min_dt_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + dt_flavor_stupid = PhysConst::hbar * min_dt_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; } if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { From 9703bac0abbfd77e7f94856701601b8159ef68c3 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 14:23:30 -0500 Subject: [PATCH 20/83] Make function compute_max_IMFP return a multifab with the maximum inverse mean free paths in the grid center --- Source/Evolve.H | 4 +- Source/Evolve.cpp | 119 ++++++++++++++++++++++++---------------------- Source/main.cpp | 5 +- 3 files changed, 67 insertions(+), 61 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 425c1ec..e11ee44 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,9 +26,9 @@ namespace GIdx void Initialize(); } -amrex::Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms); +MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms); -amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms, Real maximum_IMFP_abs); +amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms, const MultiFab& maximum_IMFP_abs); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index ac30445..5207e3d 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -136,7 +136,11 @@ Real compute_min_of_multifab (MultiFab const& multifab) }); } -Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ +MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ + + const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP + MultiFab mf_IMFP(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 + mf_IMFP.setVal(0.0); // If IMFP_method is 1, use the IMFPs from the input file and find the maximum absorption IMFP if (parms->IMFP_method == 1) { @@ -148,10 +152,18 @@ Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestPar max_IMFP_abs = std::max(max_IMFP_abs, parms->IMFP_abs[i][j]); } } - return max_IMFP_abs; + + for(amrex::MFIter mfi(mf_IMFP); mfi.isValid(); ++mfi){ + const amrex::Box& bx = mfi.validbox(); + const amrex::Array4& mf_IMFP_array = mf_IMFP.array(mfi); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ + mf_IMFP_array(i, j, k) = max_IMFP_abs; + }); + } + return mf_IMFP; // If IMFP_method is 2, use the NuLib table to find the maximum absorption IMFP - }else if (parms->IMFP_method == 2) + } else if (parms->IMFP_method == 2) { //Create NuLib table object using namespace nulib_private; @@ -162,9 +174,6 @@ Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestPar int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); - const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP - MultiFab mf_IMFP(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 - for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.validbox(); const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); @@ -253,17 +262,13 @@ Real compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestPar }); } + + return mf_IMFP; + } - //Calculate the reduction of mf_IMFP to calculate the max IMFP value. - //FIXME: FIXME: Do we need additional reduction over MPI ranks? - Real max_IMFP = compute_max_IMFP_from_mf(mf_IMFP); + // If IMFP_method is not 1 or 2, return a MultiFab with zeros + return mf_IMFP; - return max_IMFP; - }else - { - amrex::Error("Invalid IMFP method. Please check the input file."); - return -1; - } } /** @@ -285,67 +290,67 @@ Real compute_dt( const MultiFab& state, const FlavoredNeutrinoContainer&, const TestParams* parms, - Real maximum_IMFP_abs) + const MultiFab& maximum_IMFP_abs) { // ################################################################## - int start_comp = GIdx::rho; - int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye - MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); + // int start_comp = GIdx::rho; + // int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye + // MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); - const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP - MultiFab multifab_trace_n(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 + // const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP + // MultiFab multifab_trace_n(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 - for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ + // for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ - const amrex::Box& bx = mfi.validbox(); - const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); - const amrex::Array4& multifab_trace_n_array = multifab_trace_n.array(mfi); + // const amrex::Box& bx = mfi.validbox(); + // const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); + // const amrex::Array4& multifab_trace_n_array = multifab_trace_n.array(mfi); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ + // amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ - Real trace_n = 0.0; - Real trace_nbar = 0.0; + // Real trace_n = 0.0; + // Real trace_nbar = 0.0; - if (parms->do_periodic_empty_bc == 1) { + // if (parms->do_periodic_empty_bc == 1) { - if (i == 0 || i == parms->ncell[0] - 1 || - j == 0 || j == parms->ncell[1] - 1 || - k == 0 || k == parms->ncell[2] - 1) { + // if (i == 0 || i == parms->ncell[0] - 1 || + // j == 0 || j == parms->ncell[1] - 1 || + // k == 0 || k == parms->ncell[2] - 1) { - multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); - return; + // multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); + // return; - } - } + // } + // } - if (parms->do_blackhole == 1) { + // if (parms->do_blackhole == 1) { - double cell_size_x = parms->Lx / parms->ncell[0]; - double cell_size_y = parms->Ly / parms->ncell[1]; - double cell_size_z = parms->Lz / parms->ncell[2]; + // double cell_size_x = parms->Lx / parms->ncell[0]; + // double cell_size_y = parms->Ly / parms->ncell[1]; + // double cell_size_z = parms->Lz / parms->ncell[2]; - double x_cell_center = (i + 0.5) * cell_size_x; - double y_cell_center = (j + 0.5) * cell_size_y; - double z_cell_center = (k + 0.5) * cell_size_z; + // double x_cell_center = (i + 0.5) * cell_size_x; + // double y_cell_center = (j + 0.5) * cell_size_y; + // double z_cell_center = (k + 0.5) * cell_size_z; - Real distance_from_bh = sqrt(pow(x_cell_center - parms->bh_center_x, 2) + pow(y_cell_center - parms->bh_center_y, 2) + pow(z_cell_center - parms->bh_center_z, 2)); + // Real distance_from_bh = sqrt(pow(x_cell_center - parms->bh_center_x, 2) + pow(y_cell_center - parms->bh_center_y, 2) + pow(z_cell_center - parms->bh_center_z, 2)); - if (distance_from_bh < parms->bh_radius) { - multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); - return; - } - } + // if (distance_from_bh < parms->bh_radius) { + // multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); + // return; + // } + // } - #include "generated_files/Evolve.cpp_compute_trace" - multifab_trace_n_array(i, j, k) = max(trace_n, trace_nbar); + // #include "generated_files/Evolve.cpp_compute_trace" + // multifab_trace_n_array(i, j, k) = max(trace_n, trace_nbar); - }); - } + // }); + // } - Real min_trace = compute_min_of_multifab(multifab_trace_n); - printf("min_trace = %g\n", min_trace); + // Real min_trace = compute_min_of_multifab(multifab_trace_n); + // printf("min_trace = %g\n", min_trace); // ################################################################## @@ -448,8 +453,8 @@ Real compute_dt( if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { // Calculate dt_flavor_absorption - dt_flavor_absorption = (1 / (PhysConst::c * maximum_IMFP_abs)) * parms->collision_cfl_factor; - dt_flavor_absorption *= min_trace; + dt_flavor_absorption = (1 / (PhysConst::c * 1.0 )) * parms->collision_cfl_factor; + dt_flavor_absorption *= 1.0; } // pick the appropriate timestep diff --git a/Source/main.cpp b/Source/main.cpp index faf7735..6143141 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -140,10 +140,11 @@ void evolve_flavor(const TestParams* parms) } // Compute the maximum Inverse Mean Free Path (IMFP) in the domain - Real max_IMFP_absortion = 0.0; + MultiFab max_IMFP_absortion(ba, dm, 1, 0); + max_IMFP_absortion.setVal(0.0); if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { max_IMFP_absortion = compute_max_IMFP(geom, state, parms); - printf("max_IMFP_absortion = %g\n", max_IMFP_absortion); //TODO: Remove comment + // printf("max_IMFP_absortion = %g\n", max_IMFP_absortion); //TODO: Remove comment } // Initialize particles on the domain From aada23968f38ec39b755236791185f47f68d24e1 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 14:33:05 -0500 Subject: [PATCH 21/83] Compute the minimum combination of TrN/Opacity*c. This quantity provides a better representation of the time derivative of N than the previous 1/Opacity*c. --- Source/Evolve.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 5207e3d..89dc5a9 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -401,12 +401,13 @@ Real compute_dt( // ################################################################## - ReduceOps reduce_op; - ReduceData reduce_data(reduce_op); + ReduceOps< ReduceOpMin, ReduceOpMin, ReduceOpMin > reduce_op; + ReduceData< Real , Real, Real > reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; for (MFIter mfi(state); mfi.isValid(); ++mfi) { const Box& bx = mfi.fabbox(); auto const& fab = state.array(mfi); + auto const& multifab_IMFP = maximum_IMFP_abs.array(mfi); Real V_vac_max = FlavoredNeutrinoContainer::Vvac_max; reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple @@ -424,8 +425,9 @@ Real compute_dt( Real dt_adaptive = min_trace / V_adaptive; Real dt_stupid = min_trace / V_stupid; + Real dt_absorption = min_trace / multifab_IMFP(i, j, k); - return {dt_adaptive, dt_stupid}; + return {dt_adaptive, dt_stupid, dt_absorption}; }); } @@ -433,10 +435,12 @@ Real compute_dt( auto rv = reduce_data.value(); Real min_dt_adaptive = amrex::get<0>(rv); Real min_dt_stupid = amrex::get<1>(rv); + Real min_dt_absorption = amrex::get<2>(rv); // reduce across MPI ranks ParallelDescriptor::ReduceRealMin(min_dt_adaptive); ParallelDescriptor::ReduceRealMin(min_dt_stupid ); + ParallelDescriptor::ReduceRealMin(min_dt_absorption); // ################################################################## @@ -453,8 +457,7 @@ Real compute_dt( if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { // Calculate dt_flavor_absorption - dt_flavor_absorption = (1 / (PhysConst::c * 1.0 )) * parms->collision_cfl_factor; - dt_flavor_absorption *= 1.0; + dt_flavor_absorption = ( min_dt_absorption / PhysConst::c ) * parms->collision_cfl_factor; } // pick the appropriate timestep From 8a737ea89129371ea4d05aa2690cd0d6e2829215 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 14:50:01 -0500 Subject: [PATCH 22/83] Delete commented lines and handle periodic empty boundary conditions and black holes in the function that computes the time step. --- Source/Evolve.cpp | 126 ++++++++++------------------------------------ 1 file changed, 27 insertions(+), 99 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 89dc5a9..4c1f25d 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -293,67 +293,6 @@ Real compute_dt( const MultiFab& maximum_IMFP_abs) { - // ################################################################## - - // int start_comp = GIdx::rho; - // int num_comps = 3; //We only want to get GIdx::rho, GIdx::T and GIdx::Ye - // MultiFab rho_T_ye_state(state, amrex::make_alias, start_comp, num_comps); - - // const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP - // MultiFab multifab_trace_n(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 - - // for(amrex::MFIter mfi(rho_T_ye_state); mfi.isValid(); ++mfi){ - - // const amrex::Box& bx = mfi.validbox(); - // const amrex::Array4& mf_array = rho_T_ye_state.array(mfi); - // const amrex::Array4& multifab_trace_n_array = multifab_trace_n.array(mfi); - - // amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ - - // Real trace_n = 0.0; - // Real trace_nbar = 0.0; - - // if (parms->do_periodic_empty_bc == 1) { - - // if (i == 0 || i == parms->ncell[0] - 1 || - // j == 0 || j == parms->ncell[1] - 1 || - // k == 0 || k == parms->ncell[2] - 1) { - - // multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); - // return; - - // } - // } - - // if (parms->do_blackhole == 1) { - - // double cell_size_x = parms->Lx / parms->ncell[0]; - // double cell_size_y = parms->Ly / parms->ncell[1]; - // double cell_size_z = parms->Lz / parms->ncell[2]; - - // double x_cell_center = (i + 0.5) * cell_size_x; - // double y_cell_center = (j + 0.5) * cell_size_y; - // double z_cell_center = (k + 0.5) * cell_size_z; - - // Real distance_from_bh = sqrt(pow(x_cell_center - parms->bh_center_x, 2) + pow(y_cell_center - parms->bh_center_y, 2) + pow(z_cell_center - parms->bh_center_z, 2)); - - // if (distance_from_bh < parms->bh_radius) { - // multifab_trace_n_array(i, j, k) = std::numeric_limits::max(); - // return; - // } - // } - - // #include "generated_files/Evolve.cpp_compute_trace" - // multifab_trace_n_array(i, j, k) = max(trace_n, trace_nbar); - - // }); - // } - - // Real min_trace = compute_min_of_multifab(multifab_trace_n); - // printf("min_trace = %g\n", min_trace); - - // ################################################################## - AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); // translation part of timestep limit @@ -365,41 +304,6 @@ Real compute_dt( Real dt_flavor = 0.0; if (parms->flavor_cfl_factor > 0.0 && parms->collision_cfl_factor > 0.0) { - // define the reduction operator to get the max contribution to - // the potential from matter and neutrinos - // compute "effective" potential (ergs) that produces characteristic timescale - // when multiplied by hbar - // ReduceOps reduce_op; - // ReduceData reduce_data(reduce_op); - // using ReduceTuple = typename decltype(reduce_data)::Type; - // for (MFIter mfi(state); mfi.isValid(); ++mfi) { - // const Box& bx = mfi.fabbox(); - // auto const& fab = state.array(mfi); - // reduce_op.eval(bx, reduce_data, - // [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple - // { - // Real V_adaptive=0, V_adaptive2=0, V_stupid=0; - // #include "generated_files/Evolve.cpp_compute_dt_fill" - - // Real trace_n = 0.0, trace_nbar = 0.0; - // #include "generated_files/Evolve.cpp_compute_trace" - - // V_adaptive *= max(trace_n, trace_nbar); - // V_stupid *= max(trace_n, trace_nbar); - // return {V_adaptive, V_stupid}; - // }); - // } - - // // extract the reduced values from the combined reduced data structure - // auto rv = reduce_data.value(); - // Real Vmax_adaptive = amrex::get<0>(rv) + FlavoredNeutrinoContainer::Vvac_max; - // Real Vmax_stupid = amrex::get<1>(rv) + FlavoredNeutrinoContainer::Vvac_max; - - // // reduce across MPI ranks - // ParallelDescriptor::ReduceRealMax(Vmax_adaptive); - // ParallelDescriptor::ReduceRealMax(Vmax_stupid ); - - // ################################################################## ReduceOps< ReduceOpMin, ReduceOpMin, ReduceOpMin > reduce_op; ReduceData< Real , Real, Real > reduce_data(reduce_op); @@ -412,6 +316,33 @@ Real compute_dt( reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { + + if (parms->do_periodic_empty_bc == 1) { + + if (i == 0 || i == parms->ncell[0] - 1 || + j == 0 || j == parms->ncell[1] - 1 || + k == 0 || k == parms->ncell[2] - 1) { + + return {std::numeric_limits::max(), std::numeric_limits::max(), std::numeric_limits::max()}; + + } + }else if (parms->do_blackhole == 1) { + + double cell_size_x = parms->Lx / parms->ncell[0]; + double cell_size_y = parms->Ly / parms->ncell[1]; + double cell_size_z = parms->Lz / parms->ncell[2]; + + double x_cell_center = (i + 0.5) * cell_size_x; + double y_cell_center = (j + 0.5) * cell_size_y; + double z_cell_center = (k + 0.5) * cell_size_z; + + Real distance_from_bh = sqrt(pow(x_cell_center - parms->bh_center_x, 2) + pow(y_cell_center - parms->bh_center_y, 2) + pow(z_cell_center - parms->bh_center_z, 2)); + + if (distance_from_bh < parms->bh_radius) { + return {std::numeric_limits::max(), std::numeric_limits::max(), std::numeric_limits::max()}; + } + } + Real V_adaptive=0, V_adaptive2=0, V_stupid=0; #include "generated_files/Evolve.cpp_compute_dt_fill" @@ -442,9 +373,6 @@ Real compute_dt( ParallelDescriptor::ReduceRealMin(min_dt_stupid ); ParallelDescriptor::ReduceRealMin(min_dt_absorption); - // ################################################################## - - // define the dt associated with each method Real dt_flavor_adaptive = std::numeric_limits::max(); Real dt_flavor_stupid = std::numeric_limits::max(); From 3420c238250003b99e258e6502aa305aefc4364a Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 15:17:19 -0500 Subject: [PATCH 23/83] Cleaning up and addiing comments to the compute_dt function --- Source/Evolve.cpp | 86 ++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 4c1f25d..deb27b2 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -272,34 +272,43 @@ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const Tes } /** - * @brief Computes the time step for the simulation based on various CFL factors. + * @brief Computes the time step size for the simulation based on various CFL factors and conditions. * - * This function calculates the time step for the simulation by considering the - * CFL factors for translation, flavor, and collision. It uses the maximum IMFP - * value to determine the collision time step and combines it with the translation - * and flavor time steps to find the minimum time step for the simulation. + * This function calculates the time step size considering translation, flavor, and collision CFL factors. + * It ensures that the time step is limited by the smallest of these factors to maintain stability. * * @param geom The geometry of the simulation domain. - * @param state The MultiFab containing the state variables. - * @param neutrinos The container holding the flavored neutrinos. - * @param maximum_IMFP_abs The maximum inverse mean free path for absorption. - * @return The computed time step for the simulation. + * @param state The state MultiFab containing the simulation data. + * @param neutrinos The container for flavored neutrinos. + * @param parms Pointer to the structure containing simulation parameters. + * @param maximum_IMFP_abs The MultiFab containing the maximum inverse mean free path for absorption. + * + * @return The computed time step size. + * + * @note At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0. + * @note The function handles periodic boundary conditions and black hole regions if specified in the parameters. + * @note The function performs a reduction operation to find the minimum time step across all cells and MPI ranks. */ Real compute_dt( const Geometry& geom, const MultiFab& state, - const FlavoredNeutrinoContainer&, + const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms, const MultiFab& maximum_IMFP_abs) -{ +{ + // Initialize the maximum real value + const Real max_real = std::numeric_limits::max(); - AMREX_ASSERT(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0); + AMREX_ASSERT_WITH_MESSAGE(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0, + "Error: At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0."); - // translation part of timestep limit + // Get the cell size array const auto dxi = geom.CellSizeArray(); - Real dt_translation = 0.0; + Real dt_translation = 0.0; if (parms->cfl_factor > 0.0) { - dt_translation = std::min(std::min(dxi[0], dxi[1]), dxi[2]) / PhysConst::c * parms->cfl_factor; + // Calculate the time step size based on the translation CFL factor + // dt = (min(dx,dy,dz)/c) * cfl_factor + dt_translation = std::min({dxi[0], dxi[1], dxi[2]}) / PhysConst::c * parms->cfl_factor; } Real dt_flavor = 0.0; @@ -316,30 +325,31 @@ Real compute_dt( reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { - + // Check if the cell is at the boundary or inside the black hole if (parms->do_periodic_empty_bc == 1) { - + // Check if the cell is at the boundary if (i == 0 || i == parms->ncell[0] - 1 || j == 0 || j == parms->ncell[1] - 1 || k == 0 || k == parms->ncell[2] - 1) { - - return {std::numeric_limits::max(), std::numeric_limits::max(), std::numeric_limits::max()}; - + return {max_real,max_real,max_real}; } }else if (parms->do_blackhole == 1) { + // Check if the cell is inside the black hole + // Calculate the cell size double cell_size_x = parms->Lx / parms->ncell[0]; double cell_size_y = parms->Ly / parms->ncell[1]; double cell_size_z = parms->Lz / parms->ncell[2]; - + // Calculate the cell center coordinates double x_cell_center = (i + 0.5) * cell_size_x; double y_cell_center = (j + 0.5) * cell_size_y; double z_cell_center = (k + 0.5) * cell_size_z; - + // Calculate the distance from the black hole center Real distance_from_bh = sqrt(pow(x_cell_center - parms->bh_center_x, 2) + pow(y_cell_center - parms->bh_center_y, 2) + pow(z_cell_center - parms->bh_center_z, 2)); + // Check if the cell is inside the black hole if (distance_from_bh < parms->bh_radius) { - return {std::numeric_limits::max(), std::numeric_limits::max(), std::numeric_limits::max()}; + return {max_real,max_real,max_real}; } } @@ -352,11 +362,12 @@ Real compute_dt( Real trace_n = 0.0, trace_nbar = 0.0; #include "generated_files/Evolve.cpp_compute_trace_2" + // Calculate the minimum trace between neutrinos and antineutrinos Real min_trace = min(trace_n, trace_nbar); - Real dt_adaptive = min_trace / V_adaptive; - Real dt_stupid = min_trace / V_stupid; - Real dt_absorption = min_trace / multifab_IMFP(i, j, k); + Real dt_adaptive = min_trace / std::abs(V_adaptive); // dt = min(trN,trNbar)/|V_adaptive| + Real dt_stupid = min_trace / std::abs(V_stupid); // dt = min(trN,trNbar)/|V_stupid| + Real dt_absorption = min_trace / multifab_IMFP(i, j, k); // dt = 1/IMFP return {dt_adaptive, dt_stupid, dt_absorption}; }); @@ -374,17 +385,20 @@ Real compute_dt( ParallelDescriptor::ReduceRealMin(min_dt_absorption); // define the dt associated with each method - Real dt_flavor_adaptive = std::numeric_limits::max(); - Real dt_flavor_stupid = std::numeric_limits::max(); - Real dt_flavor_absorption = std::numeric_limits::max(); // Initialize with infinity + Real dt_flavor_adaptive = max_real; + Real dt_flavor_stupid = max_real; + Real dt_flavor_absorption = max_real; // Initialize with infinity if (parms->attenuation_hamiltonians != 0) { - dt_flavor_adaptive = PhysConst::hbar * min_dt_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + // dt = min( min(trN,trNbar)/|V_adaptive| ) * hbar * flavor_cfl_factor / attenuation_hamiltonians + dt_flavor_adaptive = PhysConst::hbar * min_dt_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; + // dt = min( min(trN,trNbar)/|V_stupid| ) * hbar * flavor_cfl_factor / attenuation_hamiltonians dt_flavor_stupid = PhysConst::hbar * min_dt_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; } if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { // Calculate dt_flavor_absorption + // dt = min( min( trN, trNbar ) / IMFP ) * collision_cfl_factor / c dt_flavor_absorption = ( min_dt_absorption / PhysConst::c ) * parms->collision_cfl_factor; } @@ -398,12 +412,14 @@ Real compute_dt( Real dt = 0.0; if (dt_translation != 0.0 && dt_flavor != 0.0) { dt = std::min(dt_translation, dt_flavor); - } else if (dt_translation != 0.0) { - dt = dt_translation; - } else if (dt_flavor != 0.0) { - dt = dt_flavor; } else { - amrex::Error("Timestep selection failed, both dt_translation and dt_flavor are zero. Try using both cfl_factor and flavor_cfl_factor."); + if (dt_translation != 0.0) { + dt = dt_translation; + } else if (dt_flavor != 0.0) { + dt = dt_flavor; + } else { + amrex::Error("Timestep selection failed, both dt_translation and dt_flavor are zero. Try using both cfl_factor and flavor_cfl_factor."); + } } return dt; From 459418633c6bb1c35345bf3b9735906e73c454c8 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 15:20:14 -0500 Subject: [PATCH 24/83] Add documentation for header file of Evolve.cpp file --- Source/Evolve.H | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Source/Evolve.H b/Source/Evolve.H index e11ee44..ba35c48 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -28,6 +28,24 @@ namespace GIdx MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms); +/** + * @brief Computes the time step size for the simulation based on various CFL factors and conditions. + * + * This function calculates the time step size considering translation, flavor, and collision CFL factors. + * It ensures that the time step is limited by the smallest of these factors to maintain stability. + * + * @param geom The geometry of the simulation domain. + * @param state The state MultiFab containing the simulation data. + * @param neutrinos The container for flavored neutrinos. + * @param parms Pointer to the structure containing simulation parameters. + * @param maximum_IMFP_abs The MultiFab containing the maximum inverse mean free path for absorption. + * + * @return The computed time step size. + * + * @note At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0. + * @note The function handles periodic boundary conditions and black hole regions if specified in the parameters. + * @note The function performs a reduction operation to find the minimum time step across all cells and MPI ranks. + */ amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms, const MultiFab& maximum_IMFP_abs); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); From acbf75d7ebdd2c20fd48c58977fadec123f83f38 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 15:25:06 -0500 Subject: [PATCH 25/83] Delete unnecessary generation of two files that compute the trace. --- Scripts/symbolic_hermitians/generate_code.py | 21 +------------------- Source/Evolve.cpp | 2 +- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index d9232bf..cc13396 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -535,25 +535,6 @@ def sgn(t,var): # List that will store the code for setting the derivative of the matrices N and Nbar to zero. code = [] - # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) - for t in tails: - - # Define n and nbar matrix - N = HermitianMatrix(args.N, "mf_array(i\,j\,k\,GIdx::N{}{}_{}"+t+"-start_comp)") - # Assign the trace of n and nbar to a variable - trace_N = N.trace() - code.append(["trace_n"+t+" = " + sympy.cxxcode(sympy.simplify(trace_N)) + ";"]) - - code = [line for sublist in code for line in sublist] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_compute_trace")) - - #========================# - # Evolve.cpp_compute_trace_2 # - #========================# - - # List that will store the code for setting the derivative of the matrices N and Nbar to zero. - code = [] - # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) for t in tails: @@ -564,4 +545,4 @@ def sgn(t,var): code.append(["trace_n"+t+" = " + sympy.cxxcode(sympy.simplify(trace_N)) + ";"]) code = [line for sublist in code for line in sublist] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_compute_trace_2")) \ No newline at end of file + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_compute_trace")) \ No newline at end of file diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index deb27b2..aa9fc65 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -360,7 +360,7 @@ Real compute_dt( V_stupid += V_vac_max; Real trace_n = 0.0, trace_nbar = 0.0; - #include "generated_files/Evolve.cpp_compute_trace_2" + #include "generated_files/Evolve.cpp_compute_trace" // Calculate the minimum trace between neutrinos and antineutrinos Real min_trace = min(trace_n, trace_nbar); From 229677484be4da1434ef8d470a2bad42d42c74d0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 16:10:28 -0500 Subject: [PATCH 26/83] Setting the min(TrN, TrNbar) to one in case it is zero --- Source/Evolve.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index aa9fc65..d25b68c 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -365,6 +365,9 @@ Real compute_dt( // Calculate the minimum trace between neutrinos and antineutrinos Real min_trace = min(trace_n, trace_nbar); + // Ensure that the minimum trace is not zero + if (min_trace == 0.0) min_trace = 1.0; + Real dt_adaptive = min_trace / std::abs(V_adaptive); // dt = min(trN,trNbar)/|V_adaptive| Real dt_stupid = min_trace / std::abs(V_stupid); // dt = min(trN,trNbar)/|V_stupid| Real dt_absorption = min_trace / multifab_IMFP(i, j, k); // dt = 1/IMFP From 97e981eb65476ba14666f7c3dd0f63eda52b6398 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 16:15:46 -0500 Subject: [PATCH 27/83] Cleaning space --- Source/Evolve.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index d25b68c..b5d9182 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -304,7 +304,7 @@ Real compute_dt( // Get the cell size array const auto dxi = geom.CellSizeArray(); - Real dt_translation = 0.0; + Real dt_translation = 0.0; if (parms->cfl_factor > 0.0) { // Calculate the time step size based on the translation CFL factor // dt = (min(dx,dy,dz)/c) * cfl_factor From c1be3275bffed68ab97c45ee5d8c4b6a73e3618b Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 16:36:01 -0500 Subject: [PATCH 28/83] Cleaning up compute_max_IMFP function --- Source/Evolve.cpp | 143 ++++++++++++++++++++++------------------------ 1 file changed, 68 insertions(+), 75 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index b5d9182..16e9bf7 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -138,7 +138,7 @@ Real compute_min_of_multifab (MultiFab const& multifab) MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ - const amrex::IntVect ngrow(0, 0, 0); //We do not need ghost cells for IMFP + const amrex::IntVect ngrow(0, 0, 0); // We do not need ghost cells for IMFP MultiFab mf_IMFP(state.boxarray, state.DistributionMap(), 1, ngrow); //ncomp=1 mf_IMFP.setVal(0.0); @@ -188,87 +188,80 @@ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const Tes const Real temperature = T_grid/ (1e6*CGSUnitsConst::eV); //convert temperature from erg to MeV. - double max_IMFP_abs_local = -10000; //Some random garbage value - - if(parms->IMFP_method==2){ - Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix for nucleon absortion: diag( k_e , k_u , k_t ) - Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for nucleon absortion: diag( kbar_e , kbar_u , kbar_t ) - - //--------------------- Values from NuLib table --------------------------- - int keyerr, anyerr; - double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); - int idx_group = NULIBVAR(idx_group); - //FIXME: specify neutrino energy using the following: - // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. - - //idx_species = {0 for electron neutrino, 1 for electron antineutrino and 2 for all other heavier ones} - //electron neutrino: [0, 0] - int idx_species = 0; - double absorption_opacity, scattering_opacity; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - if (anyerr) assert(0); - #ifdef DEBUG_INTERPOLATION_TABLES - printf("(Evolve.cpp) absorption_opacity[e] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[e] interpolated = %17.6g\n", scattering_opacity); - #endif - - IMFP_abs[0][0] = absorption_opacity; - - //electron antineutrino: [1, 0] - idx_species = 1; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - if (anyerr) assert(0); - - #ifdef DEBUG_INTERPOLATION_TABLES - printf("(Evolve.cpp) absorption_opacity[a] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[a] interpolated = %17.6g\n", scattering_opacity); - #endif - - IMFP_absbar[0][0] = absorption_opacity; - - //heavier ones: muon neutrino[0,1], muon antineutruino[1,1], tau neutrino[0,2], tau antineutrino[1,2] - idx_species = 2; - NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, - keyerr, anyerr, idx_species, idx_group); - if (anyerr) assert(0); - - #ifdef DEBUG_INTERPOLATION_TABLES - printf("(Evolve.cpp) absorption_opacity[x] interpolated = %17.6g\n", absorption_opacity); - printf("(Evolve.cpp) scattering_opacity[x] interpolated = %17.6g\n", scattering_opacity); - #endif - - for (int i=1; ineutrino or 1->antineutrino - IMFP_abs[i][i] = absorption_opacity ; // ... fix it ... - IMFP_absbar[i][i] = absorption_opacity ; // ... fix it ... - } - - //Calculate max of all IMFP_abs and IMFP_absbar. - if (NUM_FLAVORS == 2) { - max_IMFP_abs_local = max4(IMFP_abs[0][0], IMFP_abs[1][1], - IMFP_absbar[0][0], IMFP_absbar[1][1]); - } else if (NUM_FLAVORS == 3) { - max_IMFP_abs_local = max6(IMFP_abs[0][0], IMFP_abs[1][1], IMFP_abs[2][2], - IMFP_absbar[0][0], IMFP_absbar[1][1], IMFP_absbar[2][2]); - } - //----------------------------------------------------------------------- - } else { - - assert(0); //Only works for IMFP_method=2 + double max_IMFP_abs_local = std::numeric_limits::lowest(); // Initialize to the lowest possible value + + Real IMFP_abs[NUM_FLAVORS][NUM_FLAVORS]; // Neutrino inverse mean free path matrix for nucleon absortion: diag( k_e , k_u , k_t ) + Real IMFP_absbar[NUM_FLAVORS][NUM_FLAVORS]; // Antineutrino inverse mean free path matrix for nucleon absortion: diag( kbar_e , kbar_u , kbar_t ) + + //--------------------- Values from NuLib table --------------------------- + int keyerr, anyerr; + double *helperVarsReal_nulib = NuLib_tabulated_obj.get_helperVarsReal_nulib(); + int idx_group = NULIBVAR(idx_group); + //FIXME: specify neutrino energy using the following: + // double neutrino_energy = p.rdata(PIdx::pupt); locate energy bin using this. + + //idx_species = {0 for electron neutrino, 1 for electron antineutrino and 2 for all other heavier ones} + //electron neutrino: [0, 0] + int idx_species = 0; + double absorption_opacity, scattering_opacity; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[e] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[e] interpolated = %17.6g\n", scattering_opacity); + #endif + + IMFP_abs[0][0] = absorption_opacity; + + //electron antineutrino: [1, 0] + idx_species = 1; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[a] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[a] interpolated = %17.6g\n", scattering_opacity); + #endif + + IMFP_absbar[0][0] = absorption_opacity; + + //heavier ones: muon neutrino[0,1], muon antineutruino[1,1], tau neutrino[0,2], tau antineutrino[1,2] + idx_species = 2; + NuLib_tabulated_obj.get_opacities(rho, temperature, Ye, absorption_opacity, scattering_opacity, + keyerr, anyerr, idx_species, idx_group); + if (anyerr) assert(0); + + #ifdef DEBUG_INTERPOLATION_TABLES + printf("(Evolve.cpp) absorption_opacity[x] interpolated = %17.6g\n", absorption_opacity); + printf("(Evolve.cpp) scattering_opacity[x] interpolated = %17.6g\n", scattering_opacity); + #endif + + for (int i=1; ineutrino or 1->antineutrino + IMFP_abs[i][i] = absorption_opacity ; // ... fix it ... + IMFP_absbar[i][i] = absorption_opacity ; // ... fix it ... } + //Calculate max of all IMFP_abs and IMFP_absbar. + if (NUM_FLAVORS == 2) { + max_IMFP_abs_local = max4(IMFP_abs[0][0], IMFP_abs[1][1], + IMFP_absbar[0][0], IMFP_absbar[1][1]); + } else if (NUM_FLAVORS == 3) { + max_IMFP_abs_local = max6(IMFP_abs[0][0], IMFP_abs[1][1], IMFP_abs[2][2], + IMFP_absbar[0][0], IMFP_absbar[1][1], IMFP_absbar[2][2]); + } + //----------------------------------------------------------------------- mf_IMFP_array(i, j, k) = max_IMFP_abs_local; - }); } - + // Return the MultiFab with the maximum IMFP values + return mf_IMFP; + } else { + // If IMFP_method is not 1 or 2, return a MultiFab with zeros return mf_IMFP; } - - // If IMFP_method is not 1 or 2, return a MultiFab with zeros - return mf_IMFP; - } /** From af7a2c8950c1ca56b16274fc2610c7bffd011d7f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 16:37:37 -0500 Subject: [PATCH 29/83] Creating documentation for compute_max_IMFP function --- Source/Evolve.H | 13 +++++++++++++ Source/Evolve.cpp | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/Source/Evolve.H b/Source/Evolve.H index ba35c48..72d5224 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,6 +26,19 @@ namespace GIdx void Initialize(); } +/** + * @brief Computes the maximum Inverse Mean Free Path (IMFP) for absorption and emission processes. + * + * This function calculates the maximum IMFP for absorption processes based on the specified method in the parameters. + * It supports two methods: + * 1. Using IMFP values from the input file. + * 2. Using a NuLib table to interpolate IMFP values. + * + * @param geom The geometry of the computational domain. + * @param state The MultiFab containing the state variables. + * @param parms Pointer to the TestParams structure containing the parameters for the computation. + * @return A MultiFab containing the maximum IMFP values. + */ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms); /** diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 16e9bf7..8c6b05f 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -136,6 +136,19 @@ Real compute_min_of_multifab (MultiFab const& multifab) }); } +/** + * @brief Computes the maximum Inverse Mean Free Path (IMFP) for absorption and emission processes. + * + * This function calculates the maximum IMFP for absorption processes based on the specified method in the parameters. + * It supports two methods: + * 1. Using IMFP values from the input file. + * 2. Using a NuLib table to interpolate IMFP values. + * + * @param geom The geometry of the computational domain. + * @param state The MultiFab containing the state variables. + * @param parms Pointer to the TestParams structure containing the parameters for the computation. + * @return A MultiFab containing the maximum IMFP values. + */ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ const amrex::IntVect ngrow(0, 0, 0); // We do not need ghost cells for IMFP From 08fc11f1079b99a6c7fd6b381ca745d0afbfdcec Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Tue, 5 Nov 2024 17:06:13 -0500 Subject: [PATCH 30/83] Set the new time step to agree with the new function that computes the time step. --- Scripts/tests/coll_inst_test.py | 4 ++-- sample_inputs/inputs_collisional_instability_test | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py index 4d22f40..5cc869f 100644 --- a/Scripts/tests/coll_inst_test.py +++ b/Scripts/tests/coll_inst_test.py @@ -43,8 +43,8 @@ t[i] = np.array(hf['t(s)'][:][0]) # Fit the exponential function ( y = a e ^ ( b x ) ) to the data - l1 = 10 # initial item for fit - l2 = 40 # last item for fit + l1 = 30 # initial item for fit + l2 = 70 # last item for fit coefficients = np.polyfit(t[l1:l2], np.log(N_avg_mag[:,0,1][l1:l2]), 1) coefficients_bar = np.polyfit(t[l1:l2], np.log(Nbar_avg_mag[:,0,1][l1:l2]), 1) a = np.exp(coefficients[1]) diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index cc56120..efd2d0e 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -4,9 +4,9 @@ perturbation_amplitude = 0.0 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 1.0 -collision_cfl_factor = 1e-3 -cfl_factor = 0.5 -flavor_cfl_factor = 0.5 +collision_cfl_factor = 1e-1 +cfl_factor = 1e-1 +flavor_cfl_factor = 1e-1 max_adaptive_speedup = 0 maxError = 1e-6 @@ -27,7 +27,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 40000 +nsteps = 1000 # Simulation end time end_time = 5.0e19 @@ -41,10 +41,10 @@ T_MeV = 7.0 Ye = 0 # Write plotfiles -write_plot_every = 500 +write_plot_every = 10 # Write particle data in plotfiles -write_plot_particles_every = 500 +write_plot_particles_every = 10 # checkpointing do_restart = 0 From dac02e91d6b1bdfeadcc5566773d7cfd5ae5f400 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 13:39:09 -0500 Subject: [PATCH 31/83] Solving issue when looping over the MultiFabs. Now it does not loop over ghost cells since the MultiFab containing the maximum IMFP for every cell does not have ghost cells. --- Source/Evolve.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 8c6b05f..9a8d3a5 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -166,13 +166,7 @@ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const Tes } } - for(amrex::MFIter mfi(mf_IMFP); mfi.isValid(); ++mfi){ - const amrex::Box& bx = mfi.validbox(); - const amrex::Array4& mf_IMFP_array = mf_IMFP.array(mfi); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k){ - mf_IMFP_array(i, j, k) = max_IMFP_abs; - }); - } + mf_IMFP.setVal(max_IMFP_abs); return mf_IMFP; // If IMFP_method is 2, use the NuLib table to find the maximum absorption IMFP @@ -324,10 +318,11 @@ Real compute_dt( ReduceData< Real , Real, Real > reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; for (MFIter mfi(state); mfi.isValid(); ++mfi) { - const Box& bx = mfi.fabbox(); + const Box& bx = mfi.validbox(); auto const& fab = state.array(mfi); auto const& multifab_IMFP = maximum_IMFP_abs.array(mfi); Real V_vac_max = FlavoredNeutrinoContainer::Vvac_max; + Real max_real = std::numeric_limits::max(); reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { @@ -372,8 +367,7 @@ Real compute_dt( Real min_trace = min(trace_n, trace_nbar); // Ensure that the minimum trace is not zero - if (min_trace == 0.0) min_trace = 1.0; - + if (min_trace < 1.0) min_trace = 1.0; Real dt_adaptive = min_trace / std::abs(V_adaptive); // dt = min(trN,trNbar)/|V_adaptive| Real dt_stupid = min_trace / std::abs(V_stupid); // dt = min(trN,trNbar)/|V_stupid| Real dt_absorption = min_trace / multifab_IMFP(i, j, k); // dt = 1/IMFP From f76d6fbd23fae41d27b87e642c2223ee3efda8b0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 13:40:53 -0500 Subject: [PATCH 32/83] Delete unnecessary lines --- Source/main.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index 6143141..1be04b4 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -140,13 +140,8 @@ void evolve_flavor(const TestParams* parms) } // Compute the maximum Inverse Mean Free Path (IMFP) in the domain - MultiFab max_IMFP_absortion(ba, dm, 1, 0); - max_IMFP_absortion.setVal(0.0); - if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { - max_IMFP_absortion = compute_max_IMFP(geom, state, parms); - // printf("max_IMFP_absortion = %g\n", max_IMFP_absortion); //TODO: Remove comment - } - + MultiFab max_IMFP_absortion = compute_max_IMFP(geom, state, parms); + // Initialize particles on the domain amrex::Print() << "Initializing particles... " << std::endl; From 1021c4ebac03ee018b498f19f12c04a3cd02d2bf Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 13:41:31 -0500 Subject: [PATCH 33/83] Update empty periodic boundary condition test --- sample_inputs/inputs_bc_periodic_init | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index bbc2ad0..31d9206 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -5,8 +5,8 @@ perturbation_amplitude = 0.0 attenuation_hamiltonians = 0 collision_cfl_factor = 1e-1 -cfl_factor = 2 -flavor_cfl_factor = 2 +cfl_factor = 1e-2 +flavor_cfl_factor = 1e-1 max_adaptive_speedup = 0 maxError = 1e-6 @@ -27,7 +27,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 30000 +nsteps = 5000 # Simulation end time end_time = 5.0e19 @@ -41,10 +41,10 @@ T_MeV = 7.0 Ye = 0 # Write plotfiles -write_plot_every = 3000 +write_plot_every = 500 # Write particle data in plotfiles -write_plot_particles_every = 3000 +write_plot_particles_every = 500 # checkpointing do_restart = 0 From aa6f913de006c39ed762f24625b42b04841679b0 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 13:52:52 -0500 Subject: [PATCH 34/83] Setting the right cfl factors for particles to equilibrium test --- sample_inputs/inputs_coll_equi_test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index b2d603d..08eae84 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -4,8 +4,8 @@ perturbation_amplitude = 1e-6 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 0 -collision_cfl_factor = 0.04 -cfl_factor = 1e10 +collision_cfl_factor = 2.0 +cfl_factor = 2.0 flavor_cfl_factor = 0.5 max_adaptive_speedup = 0 maxError = 1e-6 From ec29a9b1d8998118122662dd4e3313d508629426 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 14:05:12 -0500 Subject: [PATCH 35/83] Set up the Fermi-Dirac test for multi-energy particles with the following time step CFL factors. --- sample_inputs/inputs_fermi_dirac_test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index f980f5f..43a21b1 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -4,9 +4,9 @@ perturbation_amplitude = 1e-6 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 0 -collision_cfl_factor = 100 -cfl_factor = 100 -flavor_cfl_factor = 100 +collision_cfl_factor = 50 +cfl_factor = 50 +flavor_cfl_factor = 50 max_adaptive_speedup = 0 maxError = 1e-6 From 8f7094b013b1789cade7801f9d743d2e00ede87d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 14:08:07 -0500 Subject: [PATCH 36/83] Delete unnecessary functions --- Source/Evolve.cpp | 50 ----------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 9a8d3a5..210de3c 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -86,56 +86,6 @@ template static inline AMREX_GPU_DEVICE T max6(T a, T b, T c, T d, return max(max3(a, b, c), max3(d, e, f)); } -/** - * @brief Computes the maximum Inverse Mean Free Path (IMFP) from a given MultiFab. - * - * This function performs a parallel reduction to find the maximum value of the - * Inverse Mean Free Path (IMFP) stored in the provided MultiFab. It uses the - * ParReduce function with a maximum reduction operation. - * - * @param mf_IMFP The MultiFab containing the IMFP values. It is assumed that - * the MultiFab is properly initialized and contains valid data. - * - * @return The maximum IMFP value found in the MultiFab. - */ -Real compute_max_IMFP_from_mf (MultiFab const& mf_IMFP) -{ - auto const& ma = mf_IMFP.const_arrays(); - return ParReduce(TypeList{}, TypeList{}, - mf_IMFP, IntVect(0), // zero ghost cells - [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) - noexcept -> GpuTuple - { - Array4 const& a = ma[box_no]; - return { a(i,j,k) }; - }); -} - -/** - * @brief Computes the minimum value in a MultiFab. - * - * This function performs a parallel reduction to find the minimum value - * stored in the provided MultiFab. It uses the - * ParReduce function with a minimum reduction operation. - * - * @param multifab The MultiFab containing the values. It is assumed that - * the MultiFab is properly initialized and contains valid data. - * - * @return The minimum value found in the MultiFab. - */ -Real compute_min_of_multifab (MultiFab const& multifab) -{ - auto const& ma = multifab.const_arrays(); - return ParReduce(TypeList{}, TypeList{}, - multifab, IntVect(0), // zero ghost cells - [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) - noexcept -> GpuTuple - { - Array4 const& a = ma[box_no]; - return { a(i,j,k) }; - }); -} - /** * @brief Computes the maximum Inverse Mean Free Path (IMFP) for absorption and emission processes. * From a114cc8d3655f05e2534c8b08a51095cb194e2e5 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 14:29:09 -0500 Subject: [PATCH 37/83] Solving possible issue with zero inverse mean free path --- Source/Evolve.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 210de3c..ed2a2e0 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -320,7 +320,12 @@ Real compute_dt( if (min_trace < 1.0) min_trace = 1.0; Real dt_adaptive = min_trace / std::abs(V_adaptive); // dt = min(trN,trNbar)/|V_adaptive| Real dt_stupid = min_trace / std::abs(V_stupid); // dt = min(trN,trNbar)/|V_stupid| - Real dt_absorption = min_trace / multifab_IMFP(i, j, k); // dt = 1/IMFP + Real dt_absorption = max_real; + + // Calculate the absorption time step + if multifab_multifab_IMFP(i, j, k) > 0.0) { + dt_absorption = min_trace / multifab_IMFP(i, j, k); // dt = min(trN,trNbar)/IMFP + } return {dt_adaptive, dt_stupid, dt_absorption}; }); From 735891652d0f41a4f95b483e99585a43d280a4bf Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 14:33:19 -0500 Subject: [PATCH 38/83] Improve documentation --- Source/Evolve.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index ed2a2e0..186198d 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -89,7 +89,7 @@ template static inline AMREX_GPU_DEVICE T max6(T a, T b, T c, T d, /** * @brief Computes the maximum Inverse Mean Free Path (IMFP) for absorption and emission processes. * - * This function calculates the maximum IMFP for absorption processes based on the specified method in the parameters. + * This function calculates the maximum IMFP for absorption processes on every cell center based on the specified method in the parameters. * It supports two methods: * 1. Using IMFP values from the input file. * 2. Using a NuLib table to interpolate IMFP values. @@ -97,7 +97,7 @@ template static inline AMREX_GPU_DEVICE T max6(T a, T b, T c, T d, * @param geom The geometry of the computational domain. * @param state The MultiFab containing the state variables. * @param parms Pointer to the TestParams structure containing the parameters for the computation. - * @return A MultiFab containing the maximum IMFP values. + * @return A MultiFab containing the maximum IMFP values for each cell center. */ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms){ From 2a3525d6fc74d7b387fab8f2c59b4ee5c8b82b94 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 6 Nov 2024 17:05:06 -0500 Subject: [PATCH 39/83] Solve syntax issue --- Source/Evolve.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 186198d..c12a792 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -323,7 +323,7 @@ Real compute_dt( Real dt_absorption = max_real; // Calculate the absorption time step - if multifab_multifab_IMFP(i, j, k) > 0.0) { + if (multifab_IMFP(i, j, k) > 0.0) { dt_absorption = min_trace / multifab_IMFP(i, j, k); // dt = min(trN,trNbar)/IMFP } From 94a9bf2fedb05f9f4742ccd28e81e87b56a15257 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Thu, 7 Nov 2024 16:09:44 -0500 Subject: [PATCH 40/83] Solving issue with fermi dirac test Now the fermi dirac test does not take into account the smallest 5 energy bins. In these energy bins the neutrino emissivity if small and the simulation takes a while to archive thermal equilibrium --- .../st9_empty_particles_multi_energy.py | 10 ++++++++++ Scripts/tests/fermi_dirac_test.py | 6 +++--- sample_inputs/inputs_fermi_dirac_test | 12 ++++++------ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py index 4ec43d2..5ab6dfd 100644 --- a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py +++ b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py @@ -16,12 +16,22 @@ nphi_equator = 16 # number of direction in equator NF = 3 # number of flavors +''' +--------- NuLib table energy bins --------- # Energy bin centers extracted from NuLib table energies_center_Mev = [1, 3, 5.23824, 8.00974, 11.4415, 15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542] # Energy in Mev # Energy bin bottom extracted from NuLib table energies_bottom_Mev = [0, 2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250] # Energy bin top extracted from NuLib table energies_top_Mev = [2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250, 311.085] +''' + +# Energy bin centers extracted from NuLib table +energies_center_Mev = [15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542] # Energy in Mev +# Energy bin bottom extracted from NuLib table +energies_bottom_Mev = [13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250] +# Energy bin top extracted from NuLib table +energies_top_Mev = [18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250, 311.085] # Energies in ergs energies_center_erg = np.array(energies_center_Mev) * 1e6*amrex.eV # Energy in ergs diff --git a/Scripts/tests/fermi_dirac_test.py b/Scripts/tests/fermi_dirac_test.py index ad7a520..ded9806 100644 --- a/Scripts/tests/fermi_dirac_test.py +++ b/Scripts/tests/fermi_dirac_test.py @@ -19,11 +19,11 @@ directories = sorted(directories, key=lambda x: int(x.split("plt")[1].split(".")[0])) # Energy bin centers extracted from NuLib table -energies_center_Mev = np.array([1, 3, 5.23824, 8.00974, 11.4415, 15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542]) # Energy in Mev +energies_center_Mev = np.array([15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542]) # Energy in Mev # Energy bin bottom extracted from NuLib table -energies_bottom_Mev = np.array([0, 2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250]) +energies_bottom_Mev = np.array([13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250]) # Energy bin top extracted from NuLib table -energies_top_Mev = np.array([2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250, 311.085]) +energies_top_Mev = np.array([18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250, 311.085]) # Energies in ergs energies_center_erg = np.array(energies_center_Mev) * 1e6 * eV # Energy in ergs diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 43a21b1..0ce1e6a 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -4,9 +4,9 @@ perturbation_amplitude = 1e-6 # attenuation parameters to time derivative of N due to hamiltonians attenuation_hamiltonians = 0 -collision_cfl_factor = 50 -cfl_factor = 50 -flavor_cfl_factor = 50 +collision_cfl_factor = 55 +cfl_factor = 55 +flavor_cfl_factor = 0 max_adaptive_speedup = 0 maxError = 1e-6 @@ -27,7 +27,7 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 2000 +nsteps = 6000 # Simulation end time end_time = 5.0e9 @@ -36,10 +36,10 @@ end_time = 5.0e9 amrex.fpe_trap_invalid=1 # Write plotfiles -write_plot_every = 2000 +write_plot_every = 6000 # Write particle data in plotfiles -write_plot_particles_every = 2000 +write_plot_particles_every = 6000 # checkpointing do_restart = 0 From 3f60f2650bf1812bea6f709bb9978ea17c3ce3e6 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 09:49:40 -0500 Subject: [PATCH 41/83] Fix time step function: now calculates the time step based on a basic equation. A more advanced time step calculation will be implemented later --- Source/Evolve.cpp | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index c12a792..dbcdef7 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -272,7 +272,6 @@ Real compute_dt( auto const& fab = state.array(mfi); auto const& multifab_IMFP = maximum_IMFP_abs.array(mfi); Real V_vac_max = FlavoredNeutrinoContainer::Vvac_max; - Real max_real = std::numeric_limits::max(); reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { @@ -309,22 +308,25 @@ Real compute_dt( V_adaptive += V_vac_max; V_stupid += V_vac_max; - - Real trace_n = 0.0, trace_nbar = 0.0; - #include "generated_files/Evolve.cpp_compute_trace" - // Calculate the minimum trace between neutrinos and antineutrinos - Real min_trace = min(trace_n, trace_nbar); + V_adaptive *= parms->attenuation_hamiltonians; + V_stupid *= parms->attenuation_hamiltonians; - // Ensure that the minimum trace is not zero - if (min_trace < 1.0) min_trace = 1.0; - Real dt_adaptive = min_trace / std::abs(V_adaptive); // dt = min(trN,trNbar)/|V_adaptive| - Real dt_stupid = min_trace / std::abs(V_stupid); // dt = min(trN,trNbar)/|V_stupid| + Real dt_adaptive = max_real; + Real dt_stupid = max_real; Real dt_absorption = max_real; + Real minimum_potential_abs = 1e-10; + + // Ensure that the minimum trace is not zero + if (std::abs(V_adaptive) > minimum_potential_abs){ + Real dt_adaptive = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_adaptive) ) ; + Real dt_stupid = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_stupid ) ) ; + } + // Calculate the absorption time step - if (multifab_IMFP(i, j, k) > 0.0) { - dt_absorption = min_trace / multifab_IMFP(i, j, k); // dt = min(trN,trNbar)/IMFP + if (multifab_IMFP(i, j, k) > minimum_potential_abs) { + dt_absorption = parms->collision_cfl_factor * ( 1.0 / ( PhysConst::c * multifab_IMFP(i, j, k) ) ); } return {dt_adaptive, dt_stupid, dt_absorption}; @@ -347,17 +349,12 @@ Real compute_dt( Real dt_flavor_stupid = max_real; Real dt_flavor_absorption = max_real; // Initialize with infinity - if (parms->attenuation_hamiltonians != 0) { - // dt = min( min(trN,trNbar)/|V_adaptive| ) * hbar * flavor_cfl_factor / attenuation_hamiltonians - dt_flavor_adaptive = PhysConst::hbar * min_dt_adaptive * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; - // dt = min( min(trN,trNbar)/|V_stupid| ) * hbar * flavor_cfl_factor / attenuation_hamiltonians - dt_flavor_stupid = PhysConst::hbar * min_dt_stupid * parms->flavor_cfl_factor / parms->attenuation_hamiltonians; - } - if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { - // Calculate dt_flavor_absorption - // dt = min( min( trN, trNbar ) / IMFP ) * collision_cfl_factor / c - dt_flavor_absorption = ( min_dt_absorption / PhysConst::c ) * parms->collision_cfl_factor; + dt_flavor_absorption = min_dt_absorption; + } + if (parms->attenuation_hamiltonians != 0) { + dt_flavor_adaptive = min_dt_adaptive; + dt_flavor_stupid = min_dt_stupid; } // pick the appropriate timestep From f1c9cd748316ddbcf4a949a757c30661cab00f6f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 09:55:49 -0500 Subject: [PATCH 42/83] Adding comments on the definitions of V_stupid and V_adaptive --- Source/Evolve.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index dbcdef7..1bfb943 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -303,6 +303,10 @@ Real compute_dt( } } + // V_stupid = Max(N_ab,Nbar_ab, Ye*rho/Mp)*4.0*sqrt(2)*GF + // V_adaptive id the magnitud of vector the following vector + // |vec{H}| = | sqrt(2)*GF * ( (N_ab - Nbar_ab) + (F - Fbar) + Ye*rho/Mp ) | + Real V_adaptive=0, V_adaptive2=0, V_stupid=0; #include "generated_files/Evolve.cpp_compute_dt_fill" From 216fc2a51ccfdf5fc33e0022678451d5d660c13e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 09:56:26 -0500 Subject: [PATCH 43/83] Delete unnecessary line that computes the trace of matrices. --- Scripts/symbolic_hermitians/generate_code.py | 21 +------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/Scripts/symbolic_hermitians/generate_code.py b/Scripts/symbolic_hermitians/generate_code.py index cc13396..2e223e9 100755 --- a/Scripts/symbolic_hermitians/generate_code.py +++ b/Scripts/symbolic_hermitians/generate_code.py @@ -526,23 +526,4 @@ def sgn(t,var): code.append(dNdt.code()) code = [line for sublist in code for line in sublist] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill_zeros")) - - #========================# - # Evolve.cpp_compute_trace # - #========================# - - # List that will store the code for setting the derivative of the matrices N and Nbar to zero. - code = [] - - # Looping over neutrinos(tail: no tail) and antineutrinos(tail: bar) - for t in tails: - - # Define n and nbar matrix - N = HermitianMatrix(args.N, "fab(i\,j\,k\,GIdx::N{}{}_{}"+t+")") - # Assign the trace of n and nbar to a variable - trace_N = N.trace() - code.append(["trace_n"+t+" = " + sympy.cxxcode(sympy.simplify(trace_N)) + ";"]) - - code = [line for sublist in code for line in sublist] - write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_compute_trace")) \ No newline at end of file + write_code(code, os.path.join(args.emu_home, "Source/generated_files", "Evolve.cpp_dfdt_fill_zeros")) \ No newline at end of file From f09220bf054a57cede40fe01a47f740ad50196d9 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 10:04:06 -0500 Subject: [PATCH 44/83] Add input parameter for the minimum allowed time step in the simulation --- Source/Parameters.H | 4 +++- sample_inputs/inputs_1d_fiducial | 3 +++ sample_inputs/inputs_bc_periodic_init | 3 +++ sample_inputs/inputs_bipolar_test | 3 +++ sample_inputs/inputs_coll_equi_test | 3 +++ sample_inputs/inputs_collisional_instability_test | 3 +++ sample_inputs/inputs_fast_flavor | 3 +++ sample_inputs/inputs_fast_flavor_nonzerok | 3 +++ sample_inputs/inputs_fermi_dirac_test | 3 +++ sample_inputs/inputs_msw_test | 3 +++ 10 files changed, 30 insertions(+), 1 deletion(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index c7ff098..c6395fc 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -30,6 +30,7 @@ struct TestParams : public amrex::Gpu::Managed bool do_restart; // Flag to restart from a previous simulation std::string restart_dir; // Directory path to restart from in case do_restart is true Real maxError; + Real minimum_time_step; // Minimum time step allowed in the simulation // File name with particles' initial conditions // This includes energy, momentum, N, Nbar, phase space volume, and others. @@ -94,7 +95,8 @@ struct TestParams : public amrex::Gpu::Managed pp.get("do_restart", do_restart); pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); - + pp.get("minimum_time_step", minimum_time_step); + // File name with particles' initial conditions // This includes energy, momentum, N, Nbar, phase space volume, and others. pp.get("particle_data_filename",particle_data_filename); diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index a8148b9..d8661c1 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -29,6 +29,9 @@ max_grid_size = 16 # Number of steps to run nsteps = 1000 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 5.0e-9 diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index 31d9206..b6863c0 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -29,6 +29,9 @@ max_grid_size = 16 # Number of steps to run nsteps = 5000 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 5.0e19 diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 9781389..84dfc9a 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -29,6 +29,9 @@ max_grid_size = 64 # Number of steps to run nsteps = 200 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 1.0 diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index 08eae84..9b7b34a 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -29,6 +29,9 @@ max_grid_size = 16 # Number of steps to run nsteps = 1000 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 5.0e9 diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index efd2d0e..b7da189 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -29,6 +29,9 @@ max_grid_size = 16 # Number of steps to run nsteps = 1000 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 5.0e19 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index fdcdebe..5afde3e 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -29,6 +29,9 @@ max_grid_size = 64 # Number of steps to run nsteps = 100 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 1.0 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index a20c6f0..af20a22 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -30,6 +30,9 @@ max_grid_size = 10 # Number of steps to run nsteps = 5000 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 1.0e-10 diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 0ce1e6a..66507a9 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -29,6 +29,9 @@ max_grid_size = 16 # Number of steps to run nsteps = 6000 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 5.0e9 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 8e83796..2c41121 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -29,6 +29,9 @@ max_grid_size = 64 # Number of steps to run nsteps = 50 +# Minimum timestep (s) +minimum_time_step = 1e-15 + # Simulation end time end_time = 1.0 From 981c3e227211e439b09b453cead7e14dac0af01e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 10:18:03 -0500 Subject: [PATCH 45/83] Including the minimum time step barrier in the function that computes time steps. --- Source/Evolve.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 1bfb943..31c41b7 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -320,7 +320,7 @@ Real compute_dt( Real dt_stupid = max_real; Real dt_absorption = max_real; - Real minimum_potential_abs = 1e-10; + Real minimum_potential_abs = parms->flavor_cfl_factor * PhysConst::hbar / parms->minimum_time_step; // Ensure that the minimum trace is not zero if (std::abs(V_adaptive) > minimum_potential_abs){ @@ -329,7 +329,7 @@ Real compute_dt( } // Calculate the absorption time step - if (multifab_IMFP(i, j, k) > minimum_potential_abs) { + if ( ( PhysConst::c * multifab_IMFP(i, j, k) ) > ( parms->collision_cfl_factor / parms->minimum_time_step) ) { dt_absorption = parms->collision_cfl_factor * ( 1.0 / ( PhysConst::c * multifab_IMFP(i, j, k) ) ); } From 94af9fb7e3d9ea34bb213f3b8971304fc6acec5a Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 10:22:34 -0500 Subject: [PATCH 46/83] Including a new neutrino container to store the minimum dt for every particle variable --- Source/main.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index 1be04b4..ebea74f 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -148,6 +148,7 @@ void evolve_flavor(const TestParams* parms) // We store old-time and new-time data FlavoredNeutrinoContainer neutrinos_old(geom, dm, ba); FlavoredNeutrinoContainer neutrinos_new(geom, dm, ba); + FlavoredNeutrinoContainer neutrinos_dt(geom, dm, ba); // Track the Figure of Merit for the simulation // defined as number of particles advanced per microsecond of walltime @@ -275,7 +276,7 @@ void evolve_flavor(const TestParams* parms) // because the last deposit_to_mesh call was at either the old time (forward Euler) // or the final RK stage, if using Runge-Kutta. printf("Setting next timestep... \n"); - const Real dt = compute_dt(geom, state, neutrinos, parms, max_IMFP_absortion); + const Real dt = compute_dt(geom, state, neutrinos, neutrinos_dt, parms, max_IMFP_absortion); integrator.set_timestep(dt); //printf("current_dt = %g, dt = %g \n", current_dt, dt); printf("Done. \n"); @@ -286,8 +287,8 @@ void evolve_flavor(const TestParams* parms) integrator.set_post_timestep(post_timestep_fun); // Get a starting timestep - const Real starting_dt = compute_dt(geom, state, neutrinos_old, parms, max_IMFP_absortion); - + const Real starting_dt = compute_dt(geom, state, neutrinos_old, neutrinos_dt, parms, max_IMFP_absortion); + // Do all the science! amrex::Print() << "Starting timestepping loop... " << std::endl; From 92559376889dcf3fe1e2aba5958d8241778fcb8a Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 10:25:31 -0500 Subject: [PATCH 47/83] Delete documentation in Evolve.H header file and input the time step neutrino container in the function that compute the time step --- Source/Evolve.H | 48 +---------------------------------------------- Source/Evolve.cpp | 5 +++-- 2 files changed, 4 insertions(+), 49 deletions(-) diff --git a/Source/Evolve.H b/Source/Evolve.H index 72d5224..24ff261 100644 --- a/Source/Evolve.H +++ b/Source/Evolve.H @@ -26,60 +26,14 @@ namespace GIdx void Initialize(); } -/** - * @brief Computes the maximum Inverse Mean Free Path (IMFP) for absorption and emission processes. - * - * This function calculates the maximum IMFP for absorption processes based on the specified method in the parameters. - * It supports two methods: - * 1. Using IMFP values from the input file. - * 2. Using a NuLib table to interpolate IMFP values. - * - * @param geom The geometry of the computational domain. - * @param state The MultiFab containing the state variables. - * @param parms Pointer to the TestParams structure containing the parameters for the computation. - * @return A MultiFab containing the maximum IMFP values. - */ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const TestParams* parms); -/** - * @brief Computes the time step size for the simulation based on various CFL factors and conditions. - * - * This function calculates the time step size considering translation, flavor, and collision CFL factors. - * It ensures that the time step is limited by the smallest of these factors to maintain stability. - * - * @param geom The geometry of the simulation domain. - * @param state The state MultiFab containing the simulation data. - * @param neutrinos The container for flavored neutrinos. - * @param parms Pointer to the structure containing simulation parameters. - * @param maximum_IMFP_abs The MultiFab containing the maximum inverse mean free path for absorption. - * - * @return The computed time step size. - * - * @note At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0. - * @note The function handles periodic boundary conditions and black hole regions if specified in the parameters. - * @note The function performs a reduction operation to find the minimum time step across all cells and MPI ranks. - */ -amrex::Real compute_dt(const amrex::Geometry& geom, const MultiFab& state, const FlavoredNeutrinoContainer& neutrinos, const TestParams* parms, const MultiFab& maximum_IMFP_abs); +amrex::Real compute_dt(const amrex::Geometry& geom, MultiFab& state, FlavoredNeutrinoContainer& neutrinos, FlavoredNeutrinoContainer& neutrinos_dt, const TestParams* parms, const MultiFab& maximum_IMFP_abs); void deposit_to_mesh(const FlavoredNeutrinoContainer& neutrinos, amrex::MultiFab& state, const amrex::Geometry& geom); void interpolate_rhs_from_mesh(FlavoredNeutrinoContainer& neutrinos_rhs, const amrex::MultiFab& state, const amrex::Geometry& geom, const TestParams* parms); -/** - * @brief Sets the N and Nbar to zero for particles inside the black hole or boundary cells. - * - * This function iterates over all particles in the `FlavoredNeutrinoContainer` and sets N and Nbar to zero if pariticles are inside the black hole or within the boundary cells of the simulation domain. - * - * @param neutrinos Reference to the container holding the flavored neutrinos. - * @param parms Pointer to the structure containing test parameters, including black hole properties and domain dimensions. - * - * The function performs the following steps: - * - Iterates over all particles in the container. - * - Computes the distance of each particle from the black hole center. - * - Sets N and Nbar to zero if the particle is inside the black hole radius. - * - Sets N and Nbar to zero if the particle is within the boundary cells of the simulation domain. - * - */ void empty_particles_at_boundary_cells(FlavoredNeutrinoContainer& neutrinos, const TestParams* parms); #endif diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 31c41b7..2e4579a 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -241,8 +241,9 @@ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const Tes */ Real compute_dt( const Geometry& geom, - const MultiFab& state, - const FlavoredNeutrinoContainer& neutrinos, + MultiFab& state, + FlavoredNeutrinoContainer& neutrinos, + FlavoredNeutrinoContainer& neutrinos_dt, const TestParams* parms, const MultiFab& maximum_IMFP_abs) { From bd4be8677c3089b088191bb63167339f023c19b8 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 10:32:50 -0500 Subject: [PATCH 48/83] Add a flag to choose between two different methods for computing the time step. --- Source/Evolve.cpp | 193 ++++++++++++++++++++++++---------------------- 1 file changed, 101 insertions(+), 92 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 2e4579a..b05c11e 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -249,9 +249,11 @@ Real compute_dt( { // Initialize the maximum real value const Real max_real = std::numeric_limits::max(); - - AMREX_ASSERT_WITH_MESSAGE(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0, - "Error: At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0."); + + // FIXME + // TEMPORARY FLAG: PUT THIS IN IMPUT PARAMETERS + int time_step_method = 0; + // FIXME // Get the cell size array const auto dxi = geom.CellSizeArray(); @@ -261,114 +263,121 @@ Real compute_dt( // dt = (min(dx,dy,dz)/c) * cfl_factor dt_translation = std::min({dxi[0], dxi[1], dxi[2]}) / PhysConst::c * parms->cfl_factor; } - Real dt_flavor = 0.0; - if (parms->flavor_cfl_factor > 0.0 && parms->collision_cfl_factor > 0.0) { - - ReduceOps< ReduceOpMin, ReduceOpMin, ReduceOpMin > reduce_op; - ReduceData< Real , Real, Real > reduce_data(reduce_op); - using ReduceTuple = typename decltype(reduce_data)::Type; - for (MFIter mfi(state); mfi.isValid(); ++mfi) { - const Box& bx = mfi.validbox(); - auto const& fab = state.array(mfi); - auto const& multifab_IMFP = maximum_IMFP_abs.array(mfi); - Real V_vac_max = FlavoredNeutrinoContainer::Vvac_max; - reduce_op.eval(bx, reduce_data, - [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple - { - // Check if the cell is at the boundary or inside the black hole - if (parms->do_periodic_empty_bc == 1) { - // Check if the cell is at the boundary - if (i == 0 || i == parms->ncell[0] - 1 || - j == 0 || j == parms->ncell[1] - 1 || - k == 0 || k == parms->ncell[2] - 1) { - return {max_real,max_real,max_real}; - } - }else if (parms->do_blackhole == 1) { - // Check if the cell is inside the black hole - - // Calculate the cell size - double cell_size_x = parms->Lx / parms->ncell[0]; - double cell_size_y = parms->Ly / parms->ncell[1]; - double cell_size_z = parms->Lz / parms->ncell[2]; - // Calculate the cell center coordinates - double x_cell_center = (i + 0.5) * cell_size_x; - double y_cell_center = (j + 0.5) * cell_size_y; - double z_cell_center = (k + 0.5) * cell_size_z; - // Calculate the distance from the black hole center - Real distance_from_bh = sqrt(pow(x_cell_center - parms->bh_center_x, 2) + pow(y_cell_center - parms->bh_center_y, 2) + pow(z_cell_center - parms->bh_center_z, 2)); - - // Check if the cell is inside the black hole - if (distance_from_bh < parms->bh_radius) { - return {max_real,max_real,max_real}; + + if ( time_step_method == 0 ){ + + AMREX_ASSERT_WITH_MESSAGE(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0, + "Error: At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0."); + + if (parms->flavor_cfl_factor > 0.0 && parms->collision_cfl_factor > 0.0) { + + ReduceOps< ReduceOpMin, ReduceOpMin, ReduceOpMin > reduce_op; + ReduceData< Real , Real, Real > reduce_data(reduce_op); + using ReduceTuple = typename decltype(reduce_data)::Type; + for (MFIter mfi(state); mfi.isValid(); ++mfi) { + const Box& bx = mfi.validbox(); + auto const& fab = state.array(mfi); + auto const& multifab_IMFP = maximum_IMFP_abs.array(mfi); + Real V_vac_max = FlavoredNeutrinoContainer::Vvac_max; + reduce_op.eval(bx, reduce_data, + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple + { + // Check if the cell is at the boundary or inside the black hole + if (parms->do_periodic_empty_bc == 1) { + // Check if the cell is at the boundary + if (i == 0 || i == parms->ncell[0] - 1 || + j == 0 || j == parms->ncell[1] - 1 || + k == 0 || k == parms->ncell[2] - 1) { + return {max_real,max_real,max_real}; + } + }else if (parms->do_blackhole == 1) { + // Check if the cell is inside the black hole + + // Calculate the cell size + double cell_size_x = parms->Lx / parms->ncell[0]; + double cell_size_y = parms->Ly / parms->ncell[1]; + double cell_size_z = parms->Lz / parms->ncell[2]; + // Calculate the cell center coordinates + double x_cell_center = (i + 0.5) * cell_size_x; + double y_cell_center = (j + 0.5) * cell_size_y; + double z_cell_center = (k + 0.5) * cell_size_z; + // Calculate the distance from the black hole center + Real distance_from_bh = sqrt(pow(x_cell_center - parms->bh_center_x, 2) + pow(y_cell_center - parms->bh_center_y, 2) + pow(z_cell_center - parms->bh_center_z, 2)); + + // Check if the cell is inside the black hole + if (distance_from_bh < parms->bh_radius) { + return {max_real,max_real,max_real}; + } } - } - // V_stupid = Max(N_ab,Nbar_ab, Ye*rho/Mp)*4.0*sqrt(2)*GF - // V_adaptive id the magnitud of vector the following vector - // |vec{H}| = | sqrt(2)*GF * ( (N_ab - Nbar_ab) + (F - Fbar) + Ye*rho/Mp ) | + // V_stupid = Max(N_ab,Nbar_ab, Ye*rho/Mp)*4.0*sqrt(2)*GF + // V_adaptive id the magnitud of vector the following vector + // |vec{H}| = | sqrt(2)*GF * ( (N_ab - Nbar_ab) + (F - Fbar) + Ye*rho/Mp ) | - Real V_adaptive=0, V_adaptive2=0, V_stupid=0; - #include "generated_files/Evolve.cpp_compute_dt_fill" + Real V_adaptive=0, V_adaptive2=0, V_stupid=0; + #include "generated_files/Evolve.cpp_compute_dt_fill" - V_adaptive += V_vac_max; - V_stupid += V_vac_max; + V_adaptive += V_vac_max; + V_stupid += V_vac_max; - V_adaptive *= parms->attenuation_hamiltonians; - V_stupid *= parms->attenuation_hamiltonians; + V_adaptive *= parms->attenuation_hamiltonians; + V_stupid *= parms->attenuation_hamiltonians; - Real dt_adaptive = max_real; - Real dt_stupid = max_real; - Real dt_absorption = max_real; + Real dt_adaptive = max_real; + Real dt_stupid = max_real; + Real dt_absorption = max_real; - Real minimum_potential_abs = parms->flavor_cfl_factor * PhysConst::hbar / parms->minimum_time_step; + Real minimum_potential_abs = parms->flavor_cfl_factor * PhysConst::hbar / parms->minimum_time_step; - // Ensure that the minimum trace is not zero - if (std::abs(V_adaptive) > minimum_potential_abs){ - Real dt_adaptive = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_adaptive) ) ; - Real dt_stupid = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_stupid ) ) ; - } + // Ensure that the minimum trace is not zero + if (std::abs(V_adaptive) > minimum_potential_abs){ + Real dt_adaptive = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_adaptive) ) ; + Real dt_stupid = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_stupid ) ) ; + } - // Calculate the absorption time step - if ( ( PhysConst::c * multifab_IMFP(i, j, k) ) > ( parms->collision_cfl_factor / parms->minimum_time_step) ) { - dt_absorption = parms->collision_cfl_factor * ( 1.0 / ( PhysConst::c * multifab_IMFP(i, j, k) ) ); - } + // Calculate the absorption time step + if ( ( PhysConst::c * multifab_IMFP(i, j, k) ) > ( parms->collision_cfl_factor / parms->minimum_time_step) ) { + dt_absorption = parms->collision_cfl_factor * ( 1.0 / ( PhysConst::c * multifab_IMFP(i, j, k) ) ); + } - return {dt_adaptive, dt_stupid, dt_absorption}; - }); - } + return {dt_adaptive, dt_stupid, dt_absorption}; + }); + } - // extract the reduced values from the combined reduced data structure - auto rv = reduce_data.value(); - Real min_dt_adaptive = amrex::get<0>(rv); - Real min_dt_stupid = amrex::get<1>(rv); - Real min_dt_absorption = amrex::get<2>(rv); + // extract the reduced values from the combined reduced data structure + auto rv = reduce_data.value(); + Real min_dt_adaptive = amrex::get<0>(rv); + Real min_dt_stupid = amrex::get<1>(rv); + Real min_dt_absorption = amrex::get<2>(rv); - // reduce across MPI ranks - ParallelDescriptor::ReduceRealMin(min_dt_adaptive); - ParallelDescriptor::ReduceRealMin(min_dt_stupid ); - ParallelDescriptor::ReduceRealMin(min_dt_absorption); + // reduce across MPI ranks + ParallelDescriptor::ReduceRealMin(min_dt_adaptive); + ParallelDescriptor::ReduceRealMin(min_dt_stupid ); + ParallelDescriptor::ReduceRealMin(min_dt_absorption); - // define the dt associated with each method - Real dt_flavor_adaptive = max_real; - Real dt_flavor_stupid = max_real; - Real dt_flavor_absorption = max_real; // Initialize with infinity + // define the dt associated with each method + Real dt_flavor_adaptive = max_real; + Real dt_flavor_stupid = max_real; + Real dt_flavor_absorption = max_real; // Initialize with infinity - if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { - dt_flavor_absorption = min_dt_absorption; - } - if (parms->attenuation_hamiltonians != 0) { - dt_flavor_adaptive = min_dt_adaptive; - dt_flavor_stupid = min_dt_stupid; - } + if (parms->IMFP_method == 1 || parms->IMFP_method == 2) { + dt_flavor_absorption = min_dt_absorption; + } + if (parms->attenuation_hamiltonians != 0) { + dt_flavor_adaptive = min_dt_adaptive; + dt_flavor_stupid = min_dt_stupid; + } - // pick the appropriate timestep - dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); - if(parms->max_adaptive_speedup > 1) { - dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); + // pick the appropriate timestep + dt_flavor = min(dt_flavor_stupid, dt_flavor_adaptive, dt_flavor_absorption); + if(parms->max_adaptive_speedup > 1) { + dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); + } } } + Real dt = 0.0; if (dt_translation != 0.0 && dt_flavor != 0.0) { dt = std::min(dt_translation, dt_flavor); From 2835ede43ed8aae858f45ceb1226dd58b4cab717 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 11:02:48 -0500 Subject: [PATCH 49/83] Add a new method to compute the time step based on the time derivative of N and Nbar for all particles. --- Source/Evolve.cpp | 117 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 5 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index b05c11e..7d79d88 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -328,16 +328,14 @@ Real compute_dt( Real dt_stupid = max_real; Real dt_absorption = max_real; - Real minimum_potential_abs = parms->flavor_cfl_factor * PhysConst::hbar / parms->minimum_time_step; - // Ensure that the minimum trace is not zero - if (std::abs(V_adaptive) > minimum_potential_abs){ + if (std::abs(V_adaptive) > 0.0){ Real dt_adaptive = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_adaptive) ) ; Real dt_stupid = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_stupid ) ) ; } // Calculate the absorption time step - if ( ( PhysConst::c * multifab_IMFP(i, j, k) ) > ( parms->collision_cfl_factor / parms->minimum_time_step) ) { + if ( multifab_IMFP(i, j, k) > 0.0 ) { dt_absorption = parms->collision_cfl_factor * ( 1.0 / ( PhysConst::c * multifab_IMFP(i, j, k) ) ); } @@ -375,8 +373,115 @@ Real compute_dt( dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); } } - } + } else if ( time_step_method == 1 ){ + + AMREX_ASSERT_WITH_MESSAGE(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0, + "Error: At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0."); + + // Compute time step with brute force method + deposit_to_mesh(neutrinos, state, geom); + state.FillBoundary(geom.periodicity()); + neutrinos_dt.copyParticles(neutrinos, true); + interpolate_rhs_from_mesh(neutrinos_dt, state, geom, parms); + + ReduceOps reduce_op; + ReduceData reduce_data(reduce_op); + + const int lev = 0; + FNParIter pti_time_derivative(neutrinos_dt, lev); + for (FNParIter pti(neutrinos, lev); pti.isValid(); ++pti) + { + + const int np = pti.numParticles(); + const int np_dt = pti_time_derivative.numParticles(); + + FlavoredNeutrinoContainer::ParticleType* pstruct = &(pti.GetArrayOfStructs()[0]); + FlavoredNeutrinoContainer::ParticleType* pstruct_dt = &(pti_time_derivative.GetArrayOfStructs()[0]); + + Real ydot_limit = 1.0e-15; + + reduce_op.eval(np, reduce_data, + [=] AMREX_GPU_DEVICE (int i) -> Real + { + + FlavoredNeutrinoContainer::ParticleType& p = pstruct[i]; + FlavoredNeutrinoContainer::ParticleType& p_dt = pstruct_dt[i]; + + auto update_dt = [&] (int idx) { + p_dt.rdata(idx) = std::numeric_limits::max(); + if (std::abs(p_dt.rdata(idx)) > 0) { + if (p.rdata(idx) > 1.0){ + p_dt.rdata(idx) = parms->flavor_cfl_factor * std::abs( p.rdata(idx) / p_dt.rdata(idx)); + } else{ + p_dt.rdata(idx) = parms->flavor_cfl_factor * std::abs( 1.0 / p_dt.rdata(idx)); + } + } + }; + + update_dt(PIdx::N00_Rebar); + update_dt(PIdx::N01_Rebar); + update_dt(PIdx::N01_Imbar); + update_dt(PIdx::N11_Rebar); + update_dt(PIdx::N00_Re); + update_dt(PIdx::N01_Re); + update_dt(PIdx::N01_Im); + update_dt(PIdx::N11_Re); + + #if NUM_FLAVORS == 3 + update_dt(PIdx::N02_Rebar); + update_dt(PIdx::N02_Imbar); + update_dt(PIdx::N12_Rebar); + update_dt(PIdx::N12_Imbar); + update_dt(PIdx::N22_Rebar); + update_dt(PIdx::N02_Re); + update_dt(PIdx::N02_Im); + update_dt(PIdx::N12_Re); + update_dt(PIdx::N12_Im); + update_dt(PIdx::N22_Re); + #endif + + Real min_dt = std::min({ + p_dt.rdata(PIdx::N00_Rebar), + p_dt.rdata(PIdx::N01_Rebar), + p_dt.rdata(PIdx::N01_Imbar), + p_dt.rdata(PIdx::N11_Rebar), + p_dt.rdata(PIdx::N00_Re), + p_dt.rdata(PIdx::N01_Re), + p_dt.rdata(PIdx::N01_Im), + p_dt.rdata(PIdx::N11_Re) + }); + + #if NUM_FLAVORS == 3 + min_dt = std::min({ + min_dt, + p_dt.rdata(PIdx::N02_Rebar), + p_dt.rdata(PIdx::N02_Imbar), + p_dt.rdata(PIdx::N12_Rebar), + p_dt.rdata(PIdx::N12_Imbar), + p_dt.rdata(PIdx::N22_Rebar), + p_dt.rdata(PIdx::N02_Re), + p_dt.rdata(PIdx::N02_Im), + p_dt.rdata(PIdx::N12_Re), + p_dt.rdata(PIdx::N12_Im), + p_dt.rdata(PIdx::N22_Re) + }); + #endif + return min_dt; + + }); + + ++pti_time_derivative; + } + + // Extract the reduced value + Real qke_dt = amrex::get<0>(reduce_data.value()); + + // Reduce across MPI ranks + ParallelDescriptor::ReduceRealMin(qke_dt); + + Real dt_flavor = qke_dt; + } Real dt = 0.0; if (dt_translation != 0.0 && dt_flavor != 0.0) { @@ -390,6 +495,8 @@ Real compute_dt( amrex::Error("Timestep selection failed, both dt_translation and dt_flavor are zero. Try using both cfl_factor and flavor_cfl_factor."); } } + + if (dtminimum_time_step) dt = parms->minimum_time_step; return dt; } From e49ff1b7737a18f90a405c035d4b0381adf4f5d5 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 11:05:17 -0500 Subject: [PATCH 50/83] Delete unnecessary variables and comments. --- Source/Evolve.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 7d79d88..4632eee 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -378,7 +378,6 @@ Real compute_dt( AMREX_ASSERT_WITH_MESSAGE(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0, "Error: At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0."); - // Compute time step with brute force method deposit_to_mesh(neutrinos, state, geom); state.FillBoundary(geom.periodicity()); neutrinos_dt.copyParticles(neutrinos, true); @@ -398,8 +397,6 @@ Real compute_dt( FlavoredNeutrinoContainer::ParticleType* pstruct = &(pti.GetArrayOfStructs()[0]); FlavoredNeutrinoContainer::ParticleType* pstruct_dt = &(pti_time_derivative.GetArrayOfStructs()[0]); - Real ydot_limit = 1.0e-15; - reduce_op.eval(np, reduce_data, [=] AMREX_GPU_DEVICE (int i) -> Real { From d437789847272849516028f38f019c45a0898880 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 11:08:54 -0500 Subject: [PATCH 51/83] Add time_step_method flag to input parameters. --- Source/Evolve.cpp | 9 ++------- Source/Parameters.H | 4 +++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 4632eee..5e850c9 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -249,11 +249,6 @@ Real compute_dt( { // Initialize the maximum real value const Real max_real = std::numeric_limits::max(); - - // FIXME - // TEMPORARY FLAG: PUT THIS IN IMPUT PARAMETERS - int time_step_method = 0; - // FIXME // Get the cell size array const auto dxi = geom.CellSizeArray(); @@ -265,7 +260,7 @@ Real compute_dt( } Real dt_flavor = 0.0; - if ( time_step_method == 0 ){ + if ( parms->time_step_method == 0 ){ AMREX_ASSERT_WITH_MESSAGE(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0 || parms->collision_cfl_factor > 0.0, "Error: At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0."); @@ -373,7 +368,7 @@ Real compute_dt( dt_flavor = min(dt_flavor_stupid*parms->max_adaptive_speedup, dt_flavor_adaptive, dt_flavor_absorption); } } - } else if ( time_step_method == 1 ){ + } else if ( parms->time_step_method == 1 ){ AMREX_ASSERT_WITH_MESSAGE(parms->cfl_factor > 0.0 || parms->flavor_cfl_factor > 0.0, "Error: At least one of cfl_factor, flavor_cfl_factor, or collision_cfl_factor must be greater than 0.0."); diff --git a/Source/Parameters.H b/Source/Parameters.H index c6395fc..3a3dbaa 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -31,6 +31,7 @@ struct TestParams : public amrex::Gpu::Managed std::string restart_dir; // Directory path to restart from in case do_restart is true Real maxError; Real minimum_time_step; // Minimum time step allowed in the simulation + int time_step_method; // Time step method flag // File name with particles' initial conditions // This includes energy, momentum, N, Nbar, phase space volume, and others. @@ -96,7 +97,8 @@ struct TestParams : public amrex::Gpu::Managed pp.get("restart_dir", restart_dir); pp.get("maxError", maxError); pp.get("minimum_time_step", minimum_time_step); - + pp.get("time_step_method", time_step_method); + // File name with particles' initial conditions // This includes energy, momentum, N, Nbar, phase space volume, and others. pp.get("particle_data_filename",particle_data_filename); From 4873d891c2b66155111ecd017347008339597a9c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 11:10:35 -0500 Subject: [PATCH 52/83] Update input parameters. --- sample_inputs/inputs_1d_fiducial | 3 +++ sample_inputs/inputs_bc_periodic_init | 3 +++ sample_inputs/inputs_bipolar_test | 3 +++ sample_inputs/inputs_coll_equi_test | 3 +++ sample_inputs/inputs_collisional_instability_test | 3 +++ sample_inputs/inputs_fast_flavor | 3 +++ sample_inputs/inputs_fast_flavor_nonzerok | 3 +++ sample_inputs/inputs_fermi_dirac_test | 3 +++ sample_inputs/inputs_msw_test | 3 +++ 9 files changed, 27 insertions(+) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index d8661c1..0a7c894 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -32,6 +32,9 @@ nsteps = 1000 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 5.0e-9 diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index b6863c0..e19e8a0 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -32,6 +32,9 @@ nsteps = 5000 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 5.0e19 diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index 84dfc9a..d7830c6 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -32,6 +32,9 @@ nsteps = 200 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 1.0 diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index 9b7b34a..6ab5bda 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -32,6 +32,9 @@ nsteps = 1000 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 5.0e9 diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index b7da189..557184b 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -32,6 +32,9 @@ nsteps = 1000 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 5.0e19 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 5afde3e..782823e 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -32,6 +32,9 @@ nsteps = 100 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 1.0 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index af20a22..7391377 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -33,6 +33,9 @@ nsteps = 5000 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 1.0e-10 diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index 66507a9..b41d56c 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -32,6 +32,9 @@ nsteps = 6000 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 5.0e9 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 2c41121..2c858f1 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -32,6 +32,9 @@ nsteps = 50 # Minimum timestep (s) minimum_time_step = 1e-15 +# Time step method +time_step_method = 0 + # Simulation end time end_time = 1.0 From f5e9e712fdc5e24ea66957a463828ae27fdb432e Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 11:12:52 -0500 Subject: [PATCH 53/83] Improve script that compute dE and antineutrino chemical potential for collisional flavor instability test --- Scripts/collisions/compute_dE_coll_inst_test.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Scripts/collisions/compute_dE_coll_inst_test.py b/Scripts/collisions/compute_dE_coll_inst_test.py index dec22d2..4f4ae1c 100644 --- a/Scripts/collisions/compute_dE_coll_inst_test.py +++ b/Scripts/collisions/compute_dE_coll_inst_test.py @@ -18,16 +18,19 @@ hc = h * c # erg cm # Simulation parameters -V = 1 # Volume of a cell ( ccm ) +V = (1.0e3)**3 # Volume of a cell ( ccm ) Ndir = 92 # Number of momentum beams isotropically distributed per cell E = 20.0 # Neutrinos and antineutrinos energy bin center ( Mev ) T = 7.0 # Background matter temperature ( Mev ) -N_eq_electron_neutrino = 3.260869565e+31 # Number of electron neutrinos at equilibrium +N_eq_electron_neutrino_density = 3.0e33 # Density of electron neutrinos at equilibrium (ccm) +N_eq_electron_neutrino = N_eq_electron_neutrino_density * V / Ndir # Number of electron neutrinos at equilibrium per beam +print(f'Number of electron neutrinos at equilibrium per beam = {N_eq_electron_neutrino}') u_electron_neutrino = 20.0 # Electron neutrino chemical potential ( Mev ) # Fermi-dirac distribution factor for electron neutrinos f_eq_electron_neutrinos = 1 / ( 1 + np.exp( ( E - u_electron_neutrino ) / T ) ) # adimentional +print(f'Fermi-Dirac distribution factor for electron neutrinos = {f_eq_electron_neutrinos}') # We know : # dE^3 = 3 * Neq * ( hc )^ 3 / ( dV * dOmega * feq ) @@ -47,9 +50,13 @@ dE=np.real(complex_deltaE) # Electron neutrino flavor -N_eq_electron_antineutrino = 2.717391304e+31 # Number of electron antineutrinos at equilibrium +N_eq_electron_antineutrino_density = 2.5e33 # Density of electron antineutrinos at equilibrium (ccm) +N_eq_electron_antineutrino = N_eq_electron_antineutrino_density * V / Ndir # Number of electron antineutrinos at equilibrium per beam +print(f'Number of electron antineutrinos at equilibrium per beam = {N_eq_electron_antineutrino}') +Vphase = V * ( 4 * np.pi / Ndir ) * delta_E_cubic / 3 +print(f'Phase space volume in ccm MeV^3 = {Vphase}') # Computing electron antineutrino chemical potential f_eq_electron_antineutrino = 3 * N_eq_electron_antineutrino * ( hc )**3 / ( V * ( 4 * np.pi / Ndir ) * delta_E_cubic ) u_electron_antineutrino = E - T * np.log( 1 / f_eq_electron_antineutrino - 1 ) -print(f'Electron neutrino chemical potential in MeV = {u_electron_antineutrino}') \ No newline at end of file +print(f'Electron antineutrino chemical potential in MeV = {u_electron_antineutrino}') \ No newline at end of file From f37885e7968361e5976472d9f95ff3250f2aac49 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 11:14:25 -0500 Subject: [PATCH 54/83] Setting asserts at the end of the collisional flavor instability test. In this way the test will always produce a plot that can be convinient for visualization of error --- Scripts/tests/coll_inst_test.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py index 5cc869f..bf9df32 100644 --- a/Scripts/tests/coll_inst_test.py +++ b/Scripts/tests/coll_inst_test.py @@ -150,12 +150,6 @@ def myassert(condition): rel_error_bar = np.abs( bbar - b_lsa ) / np.abs( ( bbar + b_lsa ) / 2 ) rel_error_max = 0.05 - print(f"{rel_error} ---> relative error in ImOmega : EMU") - print(f"{rel_error_bar} ---> relative error in ImOmegabar : EMU") - - myassert( rel_error < rel_error_max ) - myassert( rel_error_bar < rel_error_max ) - ###################################################################################### ###################################################################################### @@ -274,3 +268,11 @@ def QKE(t,y): plt.legend() plt.savefig('EMU_Julien_LucasLSA_Neu.pdf') plt.close() + + # Asserts + + print(f"{rel_error} ---> relative error in ImOmega : EMU") + print(f"{rel_error_bar} ---> relative error in ImOmegabar : EMU") + + myassert( rel_error < rel_error_max ) + myassert( rel_error_bar < rel_error_max ) \ No newline at end of file From 5ed145bbeb33a6a2ee3d025254442df1f86ee39d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 12:08:51 -0500 Subject: [PATCH 55/83] Fix issue with Method 1 for computing the time step --- Source/Evolve.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 5e850c9..cf7be16 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -258,6 +258,7 @@ Real compute_dt( // dt = (min(dx,dy,dz)/c) * cfl_factor dt_translation = std::min({dxi[0], dxi[1], dxi[2]}) / PhysConst::c * parms->cfl_factor; } + Real dt_flavor = 0.0; if ( parms->time_step_method == 0 ){ @@ -400,13 +401,14 @@ Real compute_dt( FlavoredNeutrinoContainer::ParticleType& p_dt = pstruct_dt[i]; auto update_dt = [&] (int idx) { - p_dt.rdata(idx) = std::numeric_limits::max(); if (std::abs(p_dt.rdata(idx)) > 0) { - if (p.rdata(idx) > 1.0){ + if ( std::abs(p.rdata(idx)) > 1.0){ p_dt.rdata(idx) = parms->flavor_cfl_factor * std::abs( p.rdata(idx) / p_dt.rdata(idx)); } else{ p_dt.rdata(idx) = parms->flavor_cfl_factor * std::abs( 1.0 / p_dt.rdata(idx)); } + }else{ + p_dt.rdata(idx) = max_real; } }; @@ -458,7 +460,6 @@ Real compute_dt( p_dt.rdata(PIdx::N22_Re) }); #endif - return min_dt; }); @@ -472,7 +473,7 @@ Real compute_dt( // Reduce across MPI ranks ParallelDescriptor::ReduceRealMin(qke_dt); - Real dt_flavor = qke_dt; + dt_flavor = qke_dt; } Real dt = 0.0; @@ -489,6 +490,7 @@ Real compute_dt( } if (dtminimum_time_step) dt = parms->minimum_time_step; + printf("dt = %g, dt_flavor = %g, dt_translation = %g\n", dt, dt_flavor, dt_translation); return dt; } From c7dbd0d56307e3a6e882be8829c557598ab7b21a Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 12:14:50 -0500 Subject: [PATCH 56/83] Set collisional flavor instability test to use Method 1 for computing the time step --- Scripts/tests/coll_inst_test.py | 2 +- sample_inputs/inputs_collisional_instability_test | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py index bf9df32..e582a62 100644 --- a/Scripts/tests/coll_inst_test.py +++ b/Scripts/tests/coll_inst_test.py @@ -44,7 +44,7 @@ # Fit the exponential function ( y = a e ^ ( b x ) ) to the data l1 = 30 # initial item for fit - l2 = 70 # last item for fit + l2 = 150 # last item for fit coefficients = np.polyfit(t[l1:l2], np.log(N_avg_mag[:,0,1][l1:l2]), 1) coefficients_bar = np.polyfit(t[l1:l2], np.log(Nbar_avg_mag[:,0,1][l1:l2]), 1) a = np.exp(coefficients[1]) diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index 557184b..5f27b07 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -6,7 +6,7 @@ attenuation_hamiltonians = 1.0 collision_cfl_factor = 1e-1 cfl_factor = 1e-1 -flavor_cfl_factor = 1e-1 +flavor_cfl_factor = 1e0 max_adaptive_speedup = 0 maxError = 1e-6 @@ -15,9 +15,9 @@ integration.rk.type = 4 # Domain size in 3D index space ncell = (1, 1, 1) -Lx = 1 # cm -Ly = 1 # cm -Lz = 1 # cm +Lx = 1e4 # cm +Ly = 1e4 # cm +Lz = 1e4 # cm # Number of particles per cell nppc = (1, 1, 1) @@ -27,13 +27,13 @@ particle_data_filename = "particle_input.dat" max_grid_size = 16 # Number of steps to run -nsteps = 1000 +nsteps = 3000 # Minimum timestep (s) minimum_time_step = 1e-15 # Time step method -time_step_method = 0 +time_step_method = 1 # Simulation end time end_time = 5.0e19 From e8f4b84e0d6e416847aab83f8ef4f265a4b8d3f5 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 13:26:05 -0500 Subject: [PATCH 57/83] Fix issue in method 0 to compute time step --- Source/Evolve.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index cf7be16..dac2704 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -326,8 +326,8 @@ Real compute_dt( // Ensure that the minimum trace is not zero if (std::abs(V_adaptive) > 0.0){ - Real dt_adaptive = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_adaptive) ) ; - Real dt_stupid = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_stupid ) ) ; + dt_adaptive = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_adaptive) ) ; + dt_stupid = parms->flavor_cfl_factor * ( PhysConst::hbar / std::abs(V_stupid ) ) ; } // Calculate the absorption time step From d6c37a40619cb4281977e3a361e9a7a9b4900bce Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 13:57:58 -0500 Subject: [PATCH 58/83] Adding comments in st9 initial condition to avoid future confusion --- .../initial_conditions/st9_empty_particles_multi_energy.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py index 5ab6dfd..a2e76e8 100644 --- a/Scripts/initial_conditions/st9_empty_particles_multi_energy.py +++ b/Scripts/initial_conditions/st9_empty_particles_multi_energy.py @@ -17,6 +17,7 @@ NF = 3 # number of flavors ''' +# Energy bins from NuLib table (No cutted) --------- NuLib table energy bins --------- # Energy bin centers extracted from NuLib table energies_center_Mev = [1, 3, 5.23824, 8.00974, 11.4415, 15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542] # Energy in Mev @@ -26,6 +27,11 @@ energies_top_Mev = [2, 4, 6.47649, 9.54299, 13.3401, 18.0418, 23.8636, 31.0725, 39.9989, 51.0519, 64.7382, 81.6853, 102.67, 128.654, 160.828, 200.668, 250, 311.085] ''' +# The following energy bins are an extraction of the NuLib table +# I have delete the first five energy bin to make the fermi-dirac test facter +# If I simulation want to run with all energy bins the user must change the energy bins manually + +# Energy bins from NuLib table (Cutted) # Energy bin centers extracted from NuLib table energies_center_Mev = [15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542] # Energy in Mev # Energy bin bottom extracted from NuLib table From 653bc788856220468f2d2c667dd29e59f6a4125c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 14:00:30 -0500 Subject: [PATCH 59/83] Adding comments to avoid future confusion in fermi-dirac test --- Scripts/tests/fermi_dirac_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Scripts/tests/fermi_dirac_test.py b/Scripts/tests/fermi_dirac_test.py index ded9806..1459000 100644 --- a/Scripts/tests/fermi_dirac_test.py +++ b/Scripts/tests/fermi_dirac_test.py @@ -18,6 +18,10 @@ # Sort the data file names by time step number directories = sorted(directories, key=lambda x: int(x.split("plt")[1].split(".")[0])) +# +# Energy bins from NuLib table (Cutted) +# The following energy bins should be the same as the st9_empty_particles_multi_energy initial condition script +# # Energy bin centers extracted from NuLib table energies_center_Mev = np.array([15.6909, 20.9527, 27.4681, 35.5357, 45.5254, 57.8951, 73.2117, 92.1775, 115.662, 144.741, 180.748, 225.334, 280.542]) # Energy in Mev # Energy bin bottom extracted from NuLib table From 40a853b15c2afe527d8f8285141833e16303ecac Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 14:03:10 -0500 Subject: [PATCH 60/83] Remove 'FIXME' comments. --- Source/Evolve.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index dac2704..e377d7f 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -197,8 +197,8 @@ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const Tes #endif for (int i=1; ineutrino or 1->antineutrino - IMFP_abs[i][i] = absorption_opacity ; // ... fix it ... - IMFP_absbar[i][i] = absorption_opacity ; // ... fix it ... + IMFP_abs[i][i] = absorption_opacity ; + IMFP_absbar[i][i] = absorption_opacity ; } //Calculate max of all IMFP_abs and IMFP_absbar. From 34f27cf60e253dfd7dd6e8243f2b032f94829f93 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 14:06:43 -0500 Subject: [PATCH 61/83] Return mf_IMFP at the end of the function instead of in every if statement. --- Source/Evolve.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index e377d7f..3c0f0c6 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -117,7 +117,6 @@ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const Tes } mf_IMFP.setVal(max_IMFP_abs); - return mf_IMFP; // If IMFP_method is 2, use the NuLib table to find the maximum absorption IMFP } else if (parms->IMFP_method == 2) @@ -213,12 +212,9 @@ MultiFab compute_max_IMFP(const Geometry& geom, const MultiFab& state, const Tes mf_IMFP_array(i, j, k) = max_IMFP_abs_local; }); } - // Return the MultiFab with the maximum IMFP values - return mf_IMFP; - } else { - // If IMFP_method is not 1 or 2, return a MultiFab with zeros - return mf_IMFP; } + // Return the MultiFab with the maximum IMFP values + return mf_IMFP; } /** From a5f4b30e8f1d53b37d9689acb2eb8d2423e075d5 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 14:36:43 -0500 Subject: [PATCH 62/83] Delete print line in main --- Source/main.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index ebea74f..75c7762 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -255,9 +255,8 @@ void evolve_flavor(const TestParams* parms) const int step = integrator.get_step_number(); const Real time = integrator.get_time(); - printf("Writing reduced data to file... \n"); + // Write the reduced data to file rd.WriteReducedData0D(geom, state, neutrinos, time, step+1); - printf("Done. \n"); run_fom += neutrinos.TotalNumberOfParticles(); From c2aa1b4a352c8308cfb938f5af48f29fd930ff18 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 14:56:22 -0500 Subject: [PATCH 63/83] Create a script to plot the polarization vector components for single particles --- Scripts/collisions/single_particle_plot.py | 100 +++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 Scripts/collisions/single_particle_plot.py diff --git a/Scripts/collisions/single_particle_plot.py b/Scripts/collisions/single_particle_plot.py new file mode 100644 index 0000000..7aec75d --- /dev/null +++ b/Scripts/collisions/single_particle_plot.py @@ -0,0 +1,100 @@ +''' +Created by Erick Urquilla. University of Tennessee Knoxville, USA. +This script is used to plot the single particle polarization vector in the x, y, and z directions. +This script works exclusively for the two flavor case. +The script reads the plt files generated with the writeparticleinfohdf5.py script. +''' + +import numpy as np +import glob +import h5py +import matplotlib.pyplot as plt + +file_names = np.array(glob.glob('plt*.h5')) +file_names = [file_name for file_name in file_names if '_reduced_data' not in file_name] +file_names.sort(key=lambda x: int(x.split('plt')[1].split('.h5')[0])) + +particle_momentum = [0,0,0,0] +with h5py.File(file_names[0], 'r') as hdf: + data = {} + for key in hdf.keys(): + data[key] = np.array(hdf.get(key)) + particle_momentum[0] = data['pupx'][0] + particle_momentum[1] = data['pupy'][0] + particle_momentum[2] = data['pupz'][0] + particle_momentum[3] = data['pupt'][0] + +time = np.zeros(len(file_names)) +N00_Re = np.zeros(len(file_names)) +N00_Rebar = np.zeros(len(file_names)) +N01_Im = np.zeros(len(file_names)) +N01_Imbar = np.zeros(len(file_names)) +N01_Re = np.zeros(len(file_names)) +N01_Rebar = np.zeros(len(file_names)) +N11_Re = np.zeros(len(file_names)) +N11_Rebar = np.zeros(len(file_names)) + +for i, file in enumerate(file_names): + with h5py.File(file, 'r') as hdf: + + # ['N00_Re', 'N00_Rebar', 'N01_Im', 'N01_Imbar', 'N01_Re', 'N01_Rebar', 'N11_Re', 'N11_Rebar', 'TrHN', 'Vphase', 'pos_x', 'pos_y', 'pos_z', 'pupt', 'pupx', 'pupy', 'pupz', 'time', 'x', 'y', 'z'] + + data = {} + for key in hdf.keys(): + data[key] = np.array(hdf.get(key)) + + for j in range(len(data['pupx'])): + if (data['pupx'][j] == particle_momentum[0] and + data['pupy'][j] == particle_momentum[1] and + data['pupz'][j] == particle_momentum[2] and + data['pupt'][j] == particle_momentum[3]): + time[i] = data['time'][j] + N00_Re[i] = data['N00_Re'][j] + N00_Rebar[i] = data['N00_Rebar'][j] + N01_Im[i] = data['N01_Im'][j] + N01_Imbar[i] = data['N01_Imbar'][j] + N01_Re[i] = data['N01_Re'][j] + N01_Rebar[i] = data['N01_Rebar'][j] + N11_Re[i] = data['N11_Re'][j] + N11_Rebar[i] = data['N11_Rebar'][j] + break + +P_x = 0.5 * N01_Re +P_y = 0.5 * N01_Im +P_z = 0.5 * (N00_Re - N11_Re) + +# Plot P_x vs P_y +plt.plot(P_x, P_y, color='gray', alpha=0.5) +sc = plt.scatter(P_x, P_y, c=time, cmap='viridis') +plt.colorbar(sc, label=r'$t \, (s)$') +plt.xlabel(r'$P_x$') +plt.ylabel(r'$P_y$') +plt.savefig('particle_momentum_plot_Px_Py.pdf') +plt.close() + +# Plot log(P_x) vs log(P_y) +plt.plot(np.log10(np.abs(P_x)), np.log10(np.abs(P_y)), color='gray', alpha=0.5) +sc = plt.scatter(np.log10(np.abs(P_x)), np.log10(np.abs(P_y)), c=time, cmap='viridis') +plt.colorbar(sc, label=r'$t \, (s)$') +plt.xlabel(r'$\log_{10}(|P_x|)$') +plt.ylabel(r'$\log_{10}(|P_y|)$') +plt.savefig('particle_momentum_plot_log_Px_Py.pdf') +plt.close() + +# Plot P_x vs P_z +plt.plot(P_x, P_z, color='gray', alpha=0.5) +sc = plt.scatter(P_x, P_z, c=time, cmap='viridis') +plt.colorbar(sc, label=r'$t \, (s)$') +plt.xlabel(r'$P_x$') +plt.ylabel(r'$P_z$') +plt.savefig('particle_momentum_plot_Px_Pz.pdf') +plt.close() + +# Plot P_y vs P_z +plt.plot(P_y, P_z, color='gray', alpha=0.5) +sc = plt.scatter(P_y, P_z, c=time, cmap='viridis') +plt.colorbar(sc, label=r'$t \, (s)$') +plt.xlabel(r'$P_y$') +plt.ylabel(r'$P_z$') +plt.savefig('particle_momentum_plot_Py_Pz.pdf') +plt.close() \ No newline at end of file From 501fe694c04a9f7935dee23a335e3163c5e0b1dd Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 15:07:25 -0500 Subject: [PATCH 64/83] Add an if statement to compute the IMFP MultiFab only when the time step method is zero --- Source/main.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Source/main.cpp b/Source/main.cpp index 75c7762..1751133 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -140,7 +140,11 @@ void evolve_flavor(const TestParams* parms) } // Compute the maximum Inverse Mean Free Path (IMFP) in the domain - MultiFab max_IMFP_absortion = compute_max_IMFP(geom, state, parms); + MultiFab max_IMFP_absortion; + max_IMFP_absortion.setVal(0.0); + if (parms->time_step_method == 0) { + max_IMFP_absortion = compute_max_IMFP(geom, state, parms); + } // Initialize particles on the domain amrex::Print() << "Initializing particles... " << std::endl; From 1652a0e90b2b27d9f63c5a6552e232dc723c73ff Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 15:44:26 -0500 Subject: [PATCH 65/83] Set the maximum grid size for every direction in the domain. --- Source/Parameters.H | 8 ++++++-- Source/main.cpp | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Source/Parameters.H b/Source/Parameters.H index 3a3dbaa..8666fd4 100644 --- a/Source/Parameters.H +++ b/Source/Parameters.H @@ -17,7 +17,9 @@ struct TestParams : public amrex::Gpu::Managed IntVect ncell; // Number of cells in the domain IntVect nppc; // Number of points of particle emission per cell in each dimension Real Lx, Ly, Lz; // Length of the domain in each dimension - int max_grid_size; // Maximum grid size subdivisions of the domain for parallelization + int max_grid_size_x; // Maximum grid size subdivisions in x of the domain for parallelization + int max_grid_size_y; // Maximum grid size subdivisions in y of the domain for parallelization + int max_grid_size_z; // Maximum grid size subdivisions in z of the domain for parallelization int nsteps; // Number of time steps Real end_time; // End time of the simulation int write_plot_every; // Write plot files every n steps @@ -84,7 +86,9 @@ struct TestParams : public amrex::Gpu::Managed pp.get("Ly", Ly); pp.get("Lz", Lz); pp.get("nppc", nppc); - pp.get("max_grid_size", max_grid_size); + pp.get("max_grid_size_x", max_grid_size_x); + pp.get("max_grid_size_y", max_grid_size_y); + pp.get("max_grid_size_z", max_grid_size_z); pp.get("nsteps", nsteps); pp.get("end_time", end_time); pp.get("cfl_factor", cfl_factor); diff --git a/Source/main.cpp b/Source/main.cpp index 1751133..3a1dd17 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -85,7 +85,9 @@ void evolve_flavor(const TestParams* parms) BoxArray ba(domain); // Break up boxarray "ba" into chunks no larger than "max_grid_size" along a direction - ba.maxSize(parms->max_grid_size); + const IntVect mgs(parms->max_grid_size_x, parms->max_grid_size_y, parms->max_grid_size_z); + ba.maxSize(mgs); + amrex::Print() << "Number of boxes created: " << ba.size() << std::endl; // This defines the physical box, [0,1] in each dimension RealBox real_box({AMREX_D_DECL( 0.0, 0.0, 0.0)}, From 61c025d793ab14427206c6c2fbf819077cbac780 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 25 Nov 2024 15:47:55 -0500 Subject: [PATCH 66/83] Update input file to define max grid size in every direction in the domain. --- sample_inputs/inputs_1d_fiducial | 4 +++- sample_inputs/inputs_bc_periodic_init | 4 +++- sample_inputs/inputs_bipolar_test | 4 +++- sample_inputs/inputs_coll_equi_test | 4 +++- sample_inputs/inputs_collisional_instability_test | 4 +++- sample_inputs/inputs_fast_flavor | 4 +++- sample_inputs/inputs_fast_flavor_nonzerok | 4 +++- sample_inputs/inputs_fermi_dirac_test | 4 +++- sample_inputs/inputs_msw_test | 4 +++- 9 files changed, 27 insertions(+), 9 deletions(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 0a7c894..5c460f5 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -24,7 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 16 +max_grid_size_x = 16 +max_grid_size_y = 1 +max_grid_size_z = 1 # Number of steps to run nsteps = 1000 diff --git a/sample_inputs/inputs_bc_periodic_init b/sample_inputs/inputs_bc_periodic_init index e19e8a0..bd5e371 100644 --- a/sample_inputs/inputs_bc_periodic_init +++ b/sample_inputs/inputs_bc_periodic_init @@ -24,7 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 16 +max_grid_size_x = 5 +max_grid_size_y = 5 +max_grid_size_z = 5 # Number of steps to run nsteps = 5000 diff --git a/sample_inputs/inputs_bipolar_test b/sample_inputs/inputs_bipolar_test index d7830c6..c5dba05 100644 --- a/sample_inputs/inputs_bipolar_test +++ b/sample_inputs/inputs_bipolar_test @@ -24,7 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 64 +max_grid_size_x = 1 +max_grid_size_y = 1 +max_grid_size_z = 1 # Number of steps to run nsteps = 200 diff --git a/sample_inputs/inputs_coll_equi_test b/sample_inputs/inputs_coll_equi_test index 6ab5bda..52d4cb7 100644 --- a/sample_inputs/inputs_coll_equi_test +++ b/sample_inputs/inputs_coll_equi_test @@ -24,7 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 16 +max_grid_size_x = 2 +max_grid_size_y = 2 +max_grid_size_z = 2 # Number of steps to run nsteps = 1000 diff --git a/sample_inputs/inputs_collisional_instability_test b/sample_inputs/inputs_collisional_instability_test index 5f27b07..96f3ea0 100644 --- a/sample_inputs/inputs_collisional_instability_test +++ b/sample_inputs/inputs_collisional_instability_test @@ -24,7 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 16 +max_grid_size_x = 1 +max_grid_size_y = 1 +max_grid_size_z = 1 # Number of steps to run nsteps = 3000 diff --git a/sample_inputs/inputs_fast_flavor b/sample_inputs/inputs_fast_flavor index 782823e..4613706 100644 --- a/sample_inputs/inputs_fast_flavor +++ b/sample_inputs/inputs_fast_flavor @@ -24,7 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 64 +max_grid_size_x = 1 +max_grid_size_y = 1 +max_grid_size_z = 1 # Number of steps to run nsteps = 100 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 7391377..5571c2e 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -25,7 +25,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 10 +max_grid_size_x = 10 +max_grid_size_y = 1 +max_grid_size_z = 1 # Number of steps to run nsteps = 5000 diff --git a/sample_inputs/inputs_fermi_dirac_test b/sample_inputs/inputs_fermi_dirac_test index b41d56c..eba04dd 100644 --- a/sample_inputs/inputs_fermi_dirac_test +++ b/sample_inputs/inputs_fermi_dirac_test @@ -24,7 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 16 +max_grid_size_x = 2 +max_grid_size_y = 2 +max_grid_size_z = 2 # Number of steps to run nsteps = 6000 diff --git a/sample_inputs/inputs_msw_test b/sample_inputs/inputs_msw_test index 2c858f1..9706c83 100644 --- a/sample_inputs/inputs_msw_test +++ b/sample_inputs/inputs_msw_test @@ -24,7 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size = 64 +max_grid_size_x = 1 +max_grid_size_y = 1 +max_grid_size_z = 1 # Number of steps to run nsteps = 50 From 86e85005221ee81515a1a87850d6dd163144360d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 27 Nov 2024 15:34:08 -0500 Subject: [PATCH 67/83] Making convertToHDF5.py script works in 3D simulations --- Scripts/data_reduction/convertToHDF5.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Scripts/data_reduction/convertToHDF5.py b/Scripts/data_reduction/convertToHDF5.py index 6489aaa..ba318fe 100755 --- a/Scripts/data_reduction/convertToHDF5.py +++ b/Scripts/data_reduction/convertToHDF5.py @@ -49,14 +49,19 @@ def convert_to_HDF5(sim_directory, DELETE_ALL_BUT_LAST_RESTART=False): if d==fluid_directories[0]: NF = eds.get_num_flavors() allData = h5py.File(sim_directory+"/allData.h5","w") + allData["dx(cm)"] = eds.dx + allData["dy(cm)"] = eds.dy allData["dz(cm)"] = eds.dz + allData["Nx"] = eds.Nx + allData["Ny"] = eds.Ny + allData["Nz"] = eds.Nz allData.create_dataset("t(s)", data=np.zeros(0), maxshape=(None,), dtype=datatype) allData.create_dataset("it", data=np.zeros(0), maxshape=(None,), dtype=int) # create fluid data sets - maxshape = (None, eds.Nz) - chunkshape = (1, eds.Nz) - zeros = np.zeros((0,eds.Nz)) + maxshape = (None,eds.Nx,eds.Ny,eds.Nz) + chunkshape = (1,eds.Nx,eds.Ny,eds.Nz) + zeros = np.zeros((0,eds.Nx,eds.Ny,eds.Nz)) varlist = [] for v in fluid_vars: for f1 in range(NF): From 0de65c0bf1a63e91259d82f9572aac8a1c70cf23 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 29 Nov 2024 16:00:55 -0500 Subject: [PATCH 68/83] Adding a line to stop execution when a NaN appears in N and N bar --- Source/Evolve.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 3c0f0c6..78a7113 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -397,6 +397,9 @@ Real compute_dt( FlavoredNeutrinoContainer::ParticleType& p_dt = pstruct_dt[i]; auto update_dt = [&] (int idx) { + if (p.rdata(idx)) { + amrex::Abort("Error: NaN value detected in rho, T_grid, or Ye."); + } if (std::abs(p_dt.rdata(idx)) > 0) { if ( std::abs(p.rdata(idx)) > 1.0){ p_dt.rdata(idx) = parms->flavor_cfl_factor * std::abs( p.rdata(idx) / p_dt.rdata(idx)); From e02dc74a22748fc01e7cb9d7ceb4a2134a6a113d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 29 Nov 2024 17:04:14 -0500 Subject: [PATCH 69/83] Improve section the leave the code is N or Nbar is NaN --- Source/Evolve.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Source/Evolve.cpp b/Source/Evolve.cpp index 78a7113..73dd740 100644 --- a/Source/Evolve.cpp +++ b/Source/Evolve.cpp @@ -397,8 +397,10 @@ Real compute_dt( FlavoredNeutrinoContainer::ParticleType& p_dt = pstruct_dt[i]; auto update_dt = [&] (int idx) { - if (p.rdata(idx)) { - amrex::Abort("Error: NaN value detected in rho, T_grid, or Ye."); + if (std::isnan(p.rdata(idx))) { + amrex::Abort("Error: NaN value detected in N or Nbar."); + p_dt.rdata(idx) = -1.0; + return; } if (std::abs(p_dt.rdata(idx)) > 0) { if ( std::abs(p.rdata(idx)) > 1.0){ @@ -475,6 +477,11 @@ Real compute_dt( dt_flavor = qke_dt; } + if (dt_flavor < 0.0) { + printf("Error: NaN value detected in N or Nbar. Aborting...\n"); + std::abort(); + } + Real dt = 0.0; if (dt_translation != 0.0 && dt_flavor != 0.0) { dt = std::min(dt_translation, dt_flavor); From 9ea0ae628dcae96e23bf4c17b12515da2b3bf55f Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 2 Dec 2024 11:34:22 -0500 Subject: [PATCH 70/83] Plotting best figures in collisional instability test --- Scripts/tests/coll_inst_test.py | 164 ++++++++++++++++++++++---------- 1 file changed, 113 insertions(+), 51 deletions(-) diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py index e582a62..cf75e93 100644 --- a/Scripts/tests/coll_inst_test.py +++ b/Scripts/tests/coll_inst_test.py @@ -21,6 +21,64 @@ parser.add_argument("-na", "--no_assert", action="store_true", help="If --no_assert is supplied, do not raise assertion errors if the test error > tolerance.") args = parser.parse_args() +############################################################ +############################################################ +# PLOT SETTINGS +import matplotlib as mpl +import matplotlib.pyplot as plt +from matplotlib.ticker import AutoLocator, AutoMinorLocator, LogLocator +import os +import cv2 +import glob + +# Font settings +mpl.rcParams['font.size'] = 22 +mpl.rcParams['font.family'] = 'serif' +mpl.rc('text', usetex=False) + +# Tick settings +mpl.rcParams['xtick.major.width'] = 2 +mpl.rcParams['xtick.major.pad'] = 8 +mpl.rcParams['xtick.minor.size'] = 4 + +mpl.rcParams['xtick.minor.width'] = 2 +mpl.rcParams['ytick.major.size'] = 7 +mpl.rcParams['ytick.major.width'] = 2 +mpl.rcParams['ytick.minor.size'] = 4 +mpl.rcParams['ytick.minor.width'] = 2 + +# Axis linewidth +mpl.rcParams['axes.linewidth'] = 2 + +# Tick direction and enabling ticks on all sides +mpl.rcParams['xtick.direction'] = 'in' +mpl.rcParams['ytick.direction'] = 'in' +mpl.rcParams['xtick.top'] = True +mpl.rcParams['ytick.right'] = True + +# Function to apply custom tick locators and other settings to an Axes object +def apply_custom_settings(ax, leg, log_scale_y=False): + + if log_scale_y: + # Use LogLocator for the y-axis if it's in log scale + ax.set_yscale('log') + ax.yaxis.set_major_locator(LogLocator(base=10.0)) + ax.yaxis.set_minor_locator(LogLocator(base=10.0, subs='auto', numticks=100)) + else: + # Use AutoLocator for regular scales + ax.yaxis.set_major_locator(AutoLocator()) + ax.yaxis.set_minor_locator(AutoMinorLocator()) + + # Apply the AutoLocator for the x-axis + ax.xaxis.set_major_locator(AutoLocator()) + ax.xaxis.set_minor_locator(AutoMinorLocator()) + + # Legend settings + leg.get_frame().set_edgecolor('w') + leg.get_frame().set_linewidth(0.0) +############################################################ +############################################################ + if __name__ == "__main__": # Create a list of data files to read @@ -54,50 +112,52 @@ print(f'{b} ---> EMU : Im Omega') print(f'{bbar} ---> EMU : Im Omegabar') - # Plots - import matplotlib.pyplot as plt - - # Plots N and Nbar - plt.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') - plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') - plt.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') - plt.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') - plt.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') - plt.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') - plt.legend() - plt.xlabel(r'$t$ (s)') - plt.ylabel(r'$N$ and $\bar{N}$') - plt.savefig('N_and_Nbar.pdf') + # Plots + fig, ax = plt.subplots() + ax.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') + ax.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') + ax.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') + ax.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') + ax.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') + ax.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') + ax.set_xlabel(r'$t$ (s)') + ax.set_ylabel(r'$N$ and $\bar{N}$') + leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) + apply_custom_settings(ax, leg, log_scale_y=False) + plt.tight_layout() + fig.savefig('N_and_Nbar.pdf', bbox_inches='tight') plt.clf() - # Plots N and F - plt.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') - plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') - plt.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') - plt.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {b}') - plt.plot(t, F_avg_mag[:,0,0,1], label = r'$F^x_{eu}$') - plt.plot(t, F_avg_mag[:,1,0,1], label = r'$F^y_{eu}$') - plt.plot(t, F_avg_mag[:,2,0,1], label = r'$F^z_{eu}$') - plt.legend() - plt.xlabel(r'$t$ (s)') - plt.ylabel(r'$N$ and $\vec{F}$') - plt.yscale('log') - plt.savefig('N_and_F.pdf') + fig, ax = plt.subplots() + ax.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') + ax.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') + ax.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') + ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {b:.2e}') + ax.plot(t, F_avg_mag[:,0,0,1], label = r'$F^x_{eu}$') + ax.plot(t, F_avg_mag[:,1,0,1], label = r'$F^y_{eu}$') + ax.plot(t, F_avg_mag[:,2,0,1], label = r'$F^z_{eu}$') + ax.set_xlabel(r'$t$ (s)') + ax.set_ylabel(r'$N$ and $\vec{F}$') + leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) + apply_custom_settings(ax, leg, log_scale_y=True) + plt.tight_layout() + fig.savefig('N_and_F.pdf', bbox_inches='tight') plt.clf() - # Plots Nbar and Fbar - plt.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') - plt.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') - plt.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') - plt.plot(t[l1:l2], Nbar_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {bbar}') - plt.plot(t, Fbar_avg_mag[:,0,0,1], label = r'$\bar{F}^x_{eu}$') - plt.plot(t, Fbar_avg_mag[:,1,0,1], label = r'$\bar{F}^y_{eu}$') - plt.plot(t, Fbar_avg_mag[:,2,0,1], label = r'$\bar{F}^z_{eu}$') - plt.legend() - plt.xlabel(r'$t$ (s)') - plt.ylabel(r'$\bar{N}$ and $\vec{\bar{F}}$') - plt.yscale('log') - plt.savefig('Nbar_and_Fbar.pdf') + fig, ax = plt.subplots() + ax.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') + ax.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') + ax.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') + ax.plot(t[l1:l2], Nbar_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {bbar:.2e}') + ax.plot(t, Fbar_avg_mag[:,0,0,1], label = r'$\bar{F}^x_{eu}$') + ax.plot(t, Fbar_avg_mag[:,1,0,1], label = r'$\bar{F}^y_{eu}$') + ax.plot(t, Fbar_avg_mag[:,2,0,1], label = r'$\bar{F}^z_{eu}$') + ax.set_xlabel(r'$t$ (s)') + ax.set_ylabel(r'$\bar{N}$ and $\vec{\bar{F}}$') + leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) + apply_custom_settings(ax, leg, log_scale_y=True) + plt.tight_layout() + fig.savefig('Nbar_and_Fbar.pdf', bbox_inches='tight') plt.clf() ###################################################################################### @@ -256,17 +316,19 @@ def QKE(t,y): print(f"{rel_error_j} ---> relative erroror in Julien script") # Plotting Julien, EMU and Lucas LSA data - plt.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$ EMU') - plt.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega EMU = {b}', linestyle = 'dashed') - plt.plot(time,N_eu_julien, label = r'$N_{eu}$ Julien script') - plt.plot(time[p1:p2],N_eu_julien[p1:p2], label = f'Im Omega Julien = {bj}', linestyle = 'dashed') - plt.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f'Im Omega Lucas LSA = {ImOmega_Lucas_LSA}', linestyle = 'dashed') - plt.xlabel(r'$t \ (\mathrm{s})$') - plt.ylabel(r'$N_{eu}$') - plt.title(f"Collisional flavor instability test") - plt.yscale('log') - plt.legend() - plt.savefig('EMU_Julien_LucasLSA_Neu.pdf') + fig, ax = plt.subplots() + ax.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$ EMU') + ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega EMU = {b:.2e}', linestyle = 'dashed') + ax.plot(time, N_eu_julien, label = r'$N_{eu}$ Julien script') + ax.plot(time[p1:p2], N_eu_julien[p1:p2], label = f'Im Omega Julien = {bj:.2e}', linestyle = 'dashed') + ax.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f'Im Omega Lucas LSA = {ImOmega_Lucas_LSA:.2e}', linestyle = 'dashed') + ax.set_xlabel(r'$t \ (\mathrm{s})$') + ax.set_ylabel(r'$N_{eu}$') + ax.set_title(f"Collisional flavor instability test") + leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) + apply_custom_settings(ax, leg, log_scale_y=True) + plt.tight_layout() + fig.savefig('EMU_Julien_LucasLSA_Neu.pdf', bbox_inches='tight') plt.close() # Asserts From 8b44a363a7fc469a36c016efadc78cdadd902224 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 2 Dec 2024 11:48:35 -0500 Subject: [PATCH 71/83] Adding units to labels and legends --- Scripts/tests/coll_inst_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py index cf75e93..298a230 100644 --- a/Scripts/tests/coll_inst_test.py +++ b/Scripts/tests/coll_inst_test.py @@ -121,7 +121,7 @@ def apply_custom_settings(ax, leg, log_scale_y=False): ax.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') ax.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') ax.set_xlabel(r'$t$ (s)') - ax.set_ylabel(r'$N$ and $\bar{N}$') + ax.set_ylabel(r'$N$ and $\bar{N}$ (cm$^{-3}$)') leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) apply_custom_settings(ax, leg, log_scale_y=False) plt.tight_layout() @@ -132,12 +132,12 @@ def apply_custom_settings(ax, leg, log_scale_y=False): ax.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') ax.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') ax.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') - ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {b:.2e}') + ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {b:.2e} s$^{{-1}}$') ax.plot(t, F_avg_mag[:,0,0,1], label = r'$F^x_{eu}$') ax.plot(t, F_avg_mag[:,1,0,1], label = r'$F^y_{eu}$') ax.plot(t, F_avg_mag[:,2,0,1], label = r'$F^z_{eu}$') ax.set_xlabel(r'$t$ (s)') - ax.set_ylabel(r'$N$ and $\vec{F}$') + ax.set_ylabel(r'$N$ and $\vec{F}$ (cm$^{-3}$)') leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) apply_custom_settings(ax, leg, log_scale_y=True) plt.tight_layout() @@ -148,12 +148,12 @@ def apply_custom_settings(ax, leg, log_scale_y=False): ax.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') ax.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') ax.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') - ax.plot(t[l1:l2], Nbar_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {bbar:.2e}') + ax.plot(t[l1:l2], Nbar_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {bbar:.2e} s$^{{-1}}$') ax.plot(t, Fbar_avg_mag[:,0,0,1], label = r'$\bar{F}^x_{eu}$') ax.plot(t, Fbar_avg_mag[:,1,0,1], label = r'$\bar{F}^y_{eu}$') ax.plot(t, Fbar_avg_mag[:,2,0,1], label = r'$\bar{F}^z_{eu}$') ax.set_xlabel(r'$t$ (s)') - ax.set_ylabel(r'$\bar{N}$ and $\vec{\bar{F}}$') + ax.set_ylabel(r'$\bar{N}$ and $\vec{\bar{F}}$ (cm$^{-3}$)') leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) apply_custom_settings(ax, leg, log_scale_y=True) plt.tight_layout() @@ -318,12 +318,12 @@ def QKE(t,y): # Plotting Julien, EMU and Lucas LSA data fig, ax = plt.subplots() ax.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$ EMU') - ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega EMU = {b:.2e}', linestyle = 'dashed') + ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega EMU = {b:.2e} s$^{{-1}}$', linestyle = 'dashed') ax.plot(time, N_eu_julien, label = r'$N_{eu}$ Julien script') - ax.plot(time[p1:p2], N_eu_julien[p1:p2], label = f'Im Omega Julien = {bj:.2e}', linestyle = 'dashed') - ax.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f'Im Omega Lucas LSA = {ImOmega_Lucas_LSA:.2e}', linestyle = 'dashed') + ax.plot(time[p1:p2], N_eu_julien[p1:p2], label = f'Im Omega Julien = {bj:.2e} s$^{{-1}}$', linestyle = 'dashed') + ax.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f'Im Omega Lucas LSA = {ImOmega_Lucas_LSA:.2e} s$^{{-1}}$', linestyle = 'dashed') ax.set_xlabel(r'$t \ (\mathrm{s})$') - ax.set_ylabel(r'$N_{eu}$') + ax.set_ylabel(r'$N_{eu}$ (cm$^{-3}$)') ax.set_title(f"Collisional flavor instability test") leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) apply_custom_settings(ax, leg, log_scale_y=True) From 104bb0f22d2d0a670ce19286ba5dd6b9e58d3910 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Mon, 2 Dec 2024 11:55:53 -0500 Subject: [PATCH 72/83] Updating plots and legens in CFI test --- Scripts/tests/coll_inst_test.py | 56 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py index 298a230..e4aff51 100644 --- a/Scripts/tests/coll_inst_test.py +++ b/Scripts/tests/coll_inst_test.py @@ -114,30 +114,30 @@ def apply_custom_settings(ax, leg, log_scale_y=False): # Plots fig, ax = plt.subplots() - ax.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') - ax.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') - ax.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') - ax.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') - ax.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') - ax.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') + ax.plot(t, N_avg_mag[:,0,0], label = r'$n_{ee}$') + ax.plot(t, N_avg_mag[:,0,1], label = r'$n_{eu}$') + ax.plot(t, N_avg_mag[:,1,1], label = r'$n_{uu}$') + ax.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{n}_{ee}$') + ax.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{n}_{eu}$') + ax.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{n}_{uu}$') ax.set_xlabel(r'$t$ (s)') - ax.set_ylabel(r'$N$ and $\bar{N}$ (cm$^{-3}$)') - leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) + ax.set_ylabel(r'$n$ and $\bar{n}$ (cm$^{-3}$)') + leg = ax.legend(framealpha=0.0, ncol=1, fontsize=15) apply_custom_settings(ax, leg, log_scale_y=False) plt.tight_layout() fig.savefig('N_and_Nbar.pdf', bbox_inches='tight') plt.clf() fig, ax = plt.subplots() - ax.plot(t, N_avg_mag[:,0,0], label = r'$N_{ee}$') - ax.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$') - ax.plot(t, N_avg_mag[:,1,1], label = r'$N_{uu}$') + ax.plot(t, N_avg_mag[:,0,0], label = r'$n_{ee}$') + ax.plot(t, N_avg_mag[:,0,1], label = r'$n_{eu}$') + ax.plot(t, N_avg_mag[:,1,1], label = r'$n_{uu}$') ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {b:.2e} s$^{{-1}}$') - ax.plot(t, F_avg_mag[:,0,0,1], label = r'$F^x_{eu}$') - ax.plot(t, F_avg_mag[:,1,0,1], label = r'$F^y_{eu}$') - ax.plot(t, F_avg_mag[:,2,0,1], label = r'$F^z_{eu}$') + ax.plot(t, F_avg_mag[:,0,0,1], label = r'$f^x_{eu}$') + ax.plot(t, F_avg_mag[:,1,0,1], label = r'$f^y_{eu}$') + ax.plot(t, F_avg_mag[:,2,0,1], label = r'$f^z_{eu}$') ax.set_xlabel(r'$t$ (s)') - ax.set_ylabel(r'$N$ and $\vec{F}$ (cm$^{-3}$)') + ax.set_ylabel(r'$n$ and $\vec{f}$ (cm$^{-3}$)') leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) apply_custom_settings(ax, leg, log_scale_y=True) plt.tight_layout() @@ -145,15 +145,15 @@ def apply_custom_settings(ax, leg, log_scale_y=False): plt.clf() fig, ax = plt.subplots() - ax.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{N}_{ee}$') - ax.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{N}_{eu}$') - ax.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{N}_{uu}$') + ax.plot(t, Nbar_avg_mag[:,0,0], label = r'$\bar{n}_{ee}$') + ax.plot(t, Nbar_avg_mag[:,0,1], label = r'$\bar{n}_{eu}$') + ax.plot(t, Nbar_avg_mag[:,1,1], label = r'$\bar{n}_{uu}$') ax.plot(t[l1:l2], Nbar_avg_mag[:,0,1][l1:l2], label = f'Im Omega = {bbar:.2e} s$^{{-1}}$') - ax.plot(t, Fbar_avg_mag[:,0,0,1], label = r'$\bar{F}^x_{eu}$') - ax.plot(t, Fbar_avg_mag[:,1,0,1], label = r'$\bar{F}^y_{eu}$') - ax.plot(t, Fbar_avg_mag[:,2,0,1], label = r'$\bar{F}^z_{eu}$') + ax.plot(t, Fbar_avg_mag[:,0,0,1], label = r'$\bar{f}^x_{eu}$') + ax.plot(t, Fbar_avg_mag[:,1,0,1], label = r'$\bar{f}^y_{eu}$') + ax.plot(t, Fbar_avg_mag[:,2,0,1], label = r'$\bar{f}^z_{eu}$') ax.set_xlabel(r'$t$ (s)') - ax.set_ylabel(r'$\bar{N}$ and $\vec{\bar{F}}$ (cm$^{-3}$)') + ax.set_ylabel(r'$\bar{n}$ and $\vec{\bar{f}}$ (cm$^{-3}$)') leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) apply_custom_settings(ax, leg, log_scale_y=True) plt.tight_layout() @@ -317,13 +317,13 @@ def QKE(t,y): # Plotting Julien, EMU and Lucas LSA data fig, ax = plt.subplots() - ax.plot(t, N_avg_mag[:,0,1], label = r'$N_{eu}$ EMU') - ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im Omega EMU = {b:.2e} s$^{{-1}}$', linestyle = 'dashed') - ax.plot(time, N_eu_julien, label = r'$N_{eu}$ Julien script') - ax.plot(time[p1:p2], N_eu_julien[p1:p2], label = f'Im Omega Julien = {bj:.2e} s$^{{-1}}$', linestyle = 'dashed') - ax.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f'Im Omega Lucas LSA = {ImOmega_Lucas_LSA:.2e} s$^{{-1}}$', linestyle = 'dashed') + ax.plot(t, N_avg_mag[:,0,1], label = r'$n_{eu}$ EMU') + ax.plot(t[l1:l2], N_avg_mag[:,0,1][l1:l2], label = f'Im$(\Omega_{{eu}})$ EMU : {b:.2e} s$^{{-1}}$', linestyle = 'dashed') + ax.plot(time, N_eu_julien, label = r'$n_{eu}$ Julien script') + ax.plot(time[p1:p2], N_eu_julien[p1:p2], label = f'Im$(\Omega_{{eu}})$ Julien F. : {bj:.2e} s$^{{-1}}$', linestyle = 'dashed') + ax.plot(time[p1:p2], 1e23*np.exp(ImOmega_Lucas_LSA*time[p1:p2]), label = f"Im$(\Omega_{{eu}})$ Luke J. (LSA) : {ImOmega_Lucas_LSA:.2e} s$^{{-1}}$", linestyle = 'dashed') ax.set_xlabel(r'$t \ (\mathrm{s})$') - ax.set_ylabel(r'$N_{eu}$ (cm$^{-3}$)') + ax.set_ylabel(r'$n_{eu}$ (cm$^{-3}$)') ax.set_title(f"Collisional flavor instability test") leg = ax.legend(framealpha=0.0, ncol=1, fontsize=12) apply_custom_settings(ax, leg, log_scale_y=True) From 8c92437252accfe9b134524755ae97249146de42 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 4 Dec 2024 11:16:55 -0500 Subject: [PATCH 73/83] Delete lines that import unused packages in the collisional instability test. --- Scripts/tests/coll_inst_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Scripts/tests/coll_inst_test.py b/Scripts/tests/coll_inst_test.py index e4aff51..a99ea22 100644 --- a/Scripts/tests/coll_inst_test.py +++ b/Scripts/tests/coll_inst_test.py @@ -7,12 +7,10 @@ import numpy as np import argparse import glob -import EmuReader import sys import os importpath = os.path.dirname(os.path.realpath(__file__))+"/../data_reduction/" sys.path.append(importpath) -import amrex_plot_tools as amrex import numpy as np import h5py import glob @@ -28,7 +26,6 @@ import matplotlib.pyplot as plt from matplotlib.ticker import AutoLocator, AutoMinorLocator, LogLocator import os -import cv2 import glob # Font settings From 4e61f3df19c14f972c97515e3a29446ec028782d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Wed, 4 Dec 2024 11:35:08 -0500 Subject: [PATCH 74/83] Setting optimal max_grid_size in tests input file --- sample_inputs/inputs_1d_fiducial | 4 ++-- sample_inputs/inputs_fast_flavor_nonzerok | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sample_inputs/inputs_1d_fiducial b/sample_inputs/inputs_1d_fiducial index 5c460f5..1eb8e86 100644 --- a/sample_inputs/inputs_1d_fiducial +++ b/sample_inputs/inputs_1d_fiducial @@ -24,9 +24,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size_x = 16 +max_grid_size_x = 1 max_grid_size_y = 1 -max_grid_size_z = 1 +max_grid_size_z = 64 # Number of steps to run nsteps = 1000 diff --git a/sample_inputs/inputs_fast_flavor_nonzerok b/sample_inputs/inputs_fast_flavor_nonzerok index 5571c2e..d7c736b 100644 --- a/sample_inputs/inputs_fast_flavor_nonzerok +++ b/sample_inputs/inputs_fast_flavor_nonzerok @@ -25,9 +25,9 @@ nppc = (1, 1, 1) particle_data_filename = "particle_input.dat" # Maximum size of each grid in the domain -max_grid_size_x = 10 +max_grid_size_x = 1 max_grid_size_y = 1 -max_grid_size_z = 1 +max_grid_size_z = 50 # Number of steps to run nsteps = 5000 From e01666d5048231144de5947dcbf8b65b73660206 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Fri, 3 Jan 2025 23:32:00 -0500 Subject: [PATCH 75/83] Add a script to save particle information from binary to HDF5, organized per cell --- .../collisions/write_particles_per_cell.py | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100755 Scripts/collisions/write_particles_per_cell.py diff --git a/Scripts/collisions/write_particles_per_cell.py b/Scripts/collisions/write_particles_per_cell.py new file mode 100755 index 0000000..ede75a5 --- /dev/null +++ b/Scripts/collisions/write_particles_per_cell.py @@ -0,0 +1,148 @@ +''' +Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. +This script was based on the script reduced_data.py. It writes all the particle information +clasifyin it per cell. The data can be found in the directories plt*_particles +The particle data is stored in hdf5 files with the following labels: cell_i_j_k.h5 +where i, j, k are the cell indexes in the x, y, z directions respectively. +i=0, j=0, k=0 is the cell with the minimum x, y, z coordinates. +''' + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/data_reduction') +import numpy as np +import glob +import multiprocessing as mp +import h5py +import amrex_plot_tools as amrex +import emu_yt_module as emu +from multiprocessing import Pool +import argparse + +class GridData(object): + def __init__(self, ad): + x = ad['index','x'].d + y = ad['index','y'].d + z = ad['index','z'].d + dx = ad['index','dx'].d + dy = ad['index','dy'].d + dz = ad['index','dz'].d + self.ad = ad + self.dx = dx[0] + self.dy = dy[0] + self.dz = dz[0] + self.xmin = np.min(x-dx/2.) + self.ymin = np.min(y-dy/2.) + self.zmin = np.min(z-dz/2.) + self.xmax = np.max(x+dx/2.) + self.ymax = np.max(y+dy/2.) + self.zmax = np.max(z+dz/2.) + self.nx = int((self.xmax - self.xmin) / self.dx + 0.5) + self.ny = int((self.ymax - self.ymin) / self.dy + 0.5) + self.nz = int((self.zmax - self.zmin) / self.dz + 0.5) + print(self.nx, self.ny, self.nz) + + def get_particle_cell_ids(self,rdata): + # get coordinates + x = rdata[:,rkey["x"]] + y = rdata[:,rkey["y"]] + z = rdata[:,rkey["z"]] + ix = (x/self.dx).astype(int) + iy = (y/self.dy).astype(int) + iz = (z/self.dz).astype(int) + + # HACK - get this grid's bounds using particle locations + ix -= np.min(ix) + iy -= np.min(iy) + iz -= np.min(iz) + nx = np.max(ix)+1 + ny = np.max(iy)+1 + nz = np.max(iz)+1 + idlist = (iz + nz*iy + nz*ny*ix).astype(int) + + return idlist + +def writehdf5files(dire): + + if not os.path.exists(dire + "_particles"): + os.makedirs(dire + "_particles") + + eds = emu.EmuDataset(dire) + t = eds.ds.current_time + ad = eds.ds.all_data() + + header = amrex.AMReXParticleHeader(dire+"/neutrinos/Header") + grid_data = GridData(ad) + nlevels = len(header.grids) + assert nlevels==1 + level = 0 + ngrids = len(header.grids[level]) + + # loop over all cells within each grid + for gridID in range(ngrids): + # read particle data on a single grid + idata, rdata = amrex.read_particle_data(dire, ptype="neutrinos", level_gridID=(level,gridID)) + + # get cell index for each particle + cell_x_index = (rdata[:,rkey['pos_x']] / grid_data.dx).astype(int) + cell_y_index = (rdata[:,rkey['pos_y']] / grid_data.dy).astype(int) + cell_z_index = (rdata[:,rkey['pos_z']] / grid_data.dz).astype(int) + cell_index = np.stack((cell_x_index, cell_y_index, cell_z_index), axis=1) + # get unique cell index + unique_cell_index = np.unique(cell_index, axis=0) + + for i in range(unique_cell_index.shape[0]): + mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) + cell_group = rdata[mask_cell_group] + cell_filename = f"{dire}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" + if os.path.exists(cell_filename): + print(f"file {cell_filename} already exists") + else: + with h5py.File(cell_filename, 'w') as cell_hf: + for label in labels: + cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) + + return dire + +# run the write hdf5 files function in parallel +if __name__ == '__main__': + + dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) + dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" + dir_particles_hdf5 = sorted(glob.glob("plt*_particles")) + dir_particles_hdf5 = [dir_particles_hdf5[i].split('_')[0] for i in range(len(dir_particles_hdf5))] + directories = [d for d in dir_particles_binary if d not in dir_particles_hdf5] + + if not directories: + print("No new directories to process.") + sys.exit(0) + + parser = argparse.ArgumentParser(description='Process some directories.') + parser.add_argument('--index', type=int, default=-2, help='index of the directory to analyze') + args = parser.parse_args() + + if args.index >= -1: + if args.index == -1: + directories = [directories[-1]] + elif args.index < len(directories): + directories = [directories[args.index]] + else: + print(f"Index {args.index} is out of range. Exiting.") + sys.exit(1) + + print(f"Directories to process: {directories}") + + # get NF + eds = emu.EmuDataset(directories[0]) + NF = eds.get_num_flavors() + if NF==2: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z', 'time', 'x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N11_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N11_Rebar', 'TrHN', 'Vphase'] + if NF==3: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z','time','x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N02_Re', 'N02_Im', 'N11_Re', 'N12_Re', 'N12_Im' ,'N22_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N02_Rebar', 'N02_Imbar', 'N11_Rebar', 'N12_Rebar' ,'N12_Imbar', 'N22_Rebar', 'TrHN', 'Vphase'] + + nproc = mp.cpu_count() + pool = Pool(nproc) + finalresult = pool.map(writehdf5files, directories) \ No newline at end of file From b403f71acaa5f3e3fc079e1a50e28a51c38a0535 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 4 Jan 2025 08:28:13 -0500 Subject: [PATCH 76/83] Optimizing script that write particles data per cell. Now it only does the clasification if the file does not exist. --- Scripts/collisions/write_particles_per_cell.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scripts/collisions/write_particles_per_cell.py b/Scripts/collisions/write_particles_per_cell.py index ede75a5..b5e7868 100755 --- a/Scripts/collisions/write_particles_per_cell.py +++ b/Scripts/collisions/write_particles_per_cell.py @@ -93,12 +93,12 @@ def writehdf5files(dire): unique_cell_index = np.unique(cell_index, axis=0) for i in range(unique_cell_index.shape[0]): - mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) - cell_group = rdata[mask_cell_group] cell_filename = f"{dire}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" if os.path.exists(cell_filename): print(f"file {cell_filename} already exists") else: + mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) + cell_group = rdata[mask_cell_group] with h5py.File(cell_filename, 'w') as cell_hf: for label in labels: cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) From e37eeb72eaf3431b51e42bfd1fb085119818192c Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 4 Jan 2025 08:59:56 -0500 Subject: [PATCH 77/83] Create a new script for parallel optimization of the write_partices_per_cell script for gpu --- .../write_particles_per_cell_gpu.py | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 Scripts/collisions/write_particles_per_cell_gpu.py diff --git a/Scripts/collisions/write_particles_per_cell_gpu.py b/Scripts/collisions/write_particles_per_cell_gpu.py new file mode 100755 index 0000000..2bb9bc9 --- /dev/null +++ b/Scripts/collisions/write_particles_per_cell_gpu.py @@ -0,0 +1,152 @@ +''' +Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. +This script was based on the script reduced_data.py. It writes all the particle information +clasifyin it per cell. The data can be found in the directories plt*_particles +The particle data is stored in hdf5 files with the following labels: cell_i_j_k.h5 +where i, j, k are the cell indexes in the x, y, z directions respectively. +i=0, j=0, k=0 is the cell with the minimum x, y, z coordinates. +''' + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/data_reduction') +import numpy as np +import glob +import multiprocessing as mp +import h5py +import amrex_plot_tools as amrex +import emu_yt_module as emu +from multiprocessing import Pool +import argparse + +class GridData(object): + def __init__(self, ad): + x = ad['index','x'].d + y = ad['index','y'].d + z = ad['index','z'].d + dx = ad['index','dx'].d + dy = ad['index','dy'].d + dz = ad['index','dz'].d + self.ad = ad + self.dx = dx[0] + self.dy = dy[0] + self.dz = dz[0] + self.xmin = np.min(x-dx/2.) + self.ymin = np.min(y-dy/2.) + self.zmin = np.min(z-dz/2.) + self.xmax = np.max(x+dx/2.) + self.ymax = np.max(y+dy/2.) + self.zmax = np.max(z+dz/2.) + self.nx = int((self.xmax - self.xmin) / self.dx + 0.5) + self.ny = int((self.ymax - self.ymin) / self.dy + 0.5) + self.nz = int((self.zmax - self.zmin) / self.dz + 0.5) + print(self.nx, self.ny, self.nz) + + def get_particle_cell_ids(self,rdata): + # get coordinates + x = rdata[:,rkey["x"]] + y = rdata[:,rkey["y"]] + z = rdata[:,rkey["z"]] + ix = (x/self.dx).astype(int) + iy = (y/self.dy).astype(int) + iz = (z/self.dz).astype(int) + + # HACK - get this grid's bounds using particle locations + ix -= np.min(ix) + iy -= np.min(iy) + iz -= np.min(iz) + nx = np.max(ix)+1 + ny = np.max(iy)+1 + nz = np.max(iz)+1 + idlist = (iz + nz*iy + nz*ny*ix).astype(int) + + return idlist + +def writehdf5files(dire): + + if not os.path.exists(dire + "_particles"): + os.makedirs(dire + "_particles") + + eds = emu.EmuDataset(dire) + t = eds.ds.current_time + ad = eds.ds.all_data() + + print(f"Processing {dire} at time {t}") + header = amrex.AMReXParticleHeader(dire+"/neutrinos/Header") + grid_data = GridData(ad) + nlevels = len(header.grids) + print(f"Number of levels: {nlevels}") + assert nlevels==1 + level = 0 + ngrids = len(header.grids[level]) + print(f"Number of grids: {ngrids}") + + # loop over all cells within each grid + for gridID in range(ngrids): + print(f"Processing grid {gridID}") + # read particle data on a single grid + idata, rdata = amrex.read_particle_data(dire, ptype="neutrinos", level_gridID=(level,gridID)) + + # get cell index for each particle + cell_x_index = (rdata[:,rkey['pos_x']] / grid_data.dx).astype(int) + cell_y_index = (rdata[:,rkey['pos_y']] / grid_data.dy).astype(int) + cell_z_index = (rdata[:,rkey['pos_z']] / grid_data.dz).astype(int) + cell_index = np.stack((cell_x_index, cell_y_index, cell_z_index), axis=1) + # get unique cell index + unique_cell_index = np.unique(cell_index, axis=0) + + for i in range(unique_cell_index.shape[0]): + cell_filename = f"{dire}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" + if os.path.exists(cell_filename): + print(f"file {cell_filename} already exists") + else: + mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) + cell_group = rdata[mask_cell_group] + with h5py.File(cell_filename, 'w') as cell_hf: + for label in labels: + cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) + + return dire + +# run the write hdf5 files function in parallel +if __name__ == '__main__': + + dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) + dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" + dir_particles_hdf5 = sorted(glob.glob("plt*_particles")) + dir_particles_hdf5 = [dir_particles_hdf5[i].split('_')[0] for i in range(len(dir_particles_hdf5))] + directories = [d for d in dir_particles_binary if d not in dir_particles_hdf5] + + if not directories: + print("No new directories to process.") + sys.exit(0) + + parser = argparse.ArgumentParser(description='Process some directories.') + parser.add_argument('--index', type=int, default=-2, help='index of the directory to analyze') + args = parser.parse_args() + + if args.index >= -1: + if args.index == -1: + directories = [directories[-1]] + elif args.index < len(directories): + directories = [directories[args.index]] + else: + print(f"Index {args.index} is out of range. Exiting.") + sys.exit(1) + + print(f"Directories to process: {directories}") + + # get NF + eds = emu.EmuDataset(directories[0]) + NF = eds.get_num_flavors() + if NF==2: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z', 'time', 'x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N11_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N11_Rebar', 'TrHN', 'Vphase'] + if NF==3: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z','time','x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N02_Re', 'N02_Im', 'N11_Re', 'N12_Re', 'N12_Im' ,'N22_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N02_Rebar', 'N02_Imbar', 'N11_Rebar', 'N12_Rebar' ,'N12_Imbar', 'N22_Rebar', 'TrHN', 'Vphase'] + + nproc = mp.cpu_count() + pool = Pool(nproc) + finalresult = pool.map(writehdf5files, directories) \ No newline at end of file From 773a93394f0c4a8e6e043b25b2cb282f40c46b5d Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 4 Jan 2025 11:03:44 -0500 Subject: [PATCH 78/83] Adding an script that write particle information parallelized over grids --- .../write_particles_per_cell_gpu.py | 152 ----------------- .../write_particles_per_cell_par_in_grid.py | 156 ++++++++++++++++++ 2 files changed, 156 insertions(+), 152 deletions(-) delete mode 100755 Scripts/collisions/write_particles_per_cell_gpu.py create mode 100755 Scripts/collisions/write_particles_per_cell_par_in_grid.py diff --git a/Scripts/collisions/write_particles_per_cell_gpu.py b/Scripts/collisions/write_particles_per_cell_gpu.py deleted file mode 100755 index 2bb9bc9..0000000 --- a/Scripts/collisions/write_particles_per_cell_gpu.py +++ /dev/null @@ -1,152 +0,0 @@ -''' -Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. -This script was based on the script reduced_data.py. It writes all the particle information -clasifyin it per cell. The data can be found in the directories plt*_particles -The particle data is stored in hdf5 files with the following labels: cell_i_j_k.h5 -where i, j, k are the cell indexes in the x, y, z directions respectively. -i=0, j=0, k=0 is the cell with the minimum x, y, z coordinates. -''' - -import os -os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' -import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/data_reduction') -import numpy as np -import glob -import multiprocessing as mp -import h5py -import amrex_plot_tools as amrex -import emu_yt_module as emu -from multiprocessing import Pool -import argparse - -class GridData(object): - def __init__(self, ad): - x = ad['index','x'].d - y = ad['index','y'].d - z = ad['index','z'].d - dx = ad['index','dx'].d - dy = ad['index','dy'].d - dz = ad['index','dz'].d - self.ad = ad - self.dx = dx[0] - self.dy = dy[0] - self.dz = dz[0] - self.xmin = np.min(x-dx/2.) - self.ymin = np.min(y-dy/2.) - self.zmin = np.min(z-dz/2.) - self.xmax = np.max(x+dx/2.) - self.ymax = np.max(y+dy/2.) - self.zmax = np.max(z+dz/2.) - self.nx = int((self.xmax - self.xmin) / self.dx + 0.5) - self.ny = int((self.ymax - self.ymin) / self.dy + 0.5) - self.nz = int((self.zmax - self.zmin) / self.dz + 0.5) - print(self.nx, self.ny, self.nz) - - def get_particle_cell_ids(self,rdata): - # get coordinates - x = rdata[:,rkey["x"]] - y = rdata[:,rkey["y"]] - z = rdata[:,rkey["z"]] - ix = (x/self.dx).astype(int) - iy = (y/self.dy).astype(int) - iz = (z/self.dz).astype(int) - - # HACK - get this grid's bounds using particle locations - ix -= np.min(ix) - iy -= np.min(iy) - iz -= np.min(iz) - nx = np.max(ix)+1 - ny = np.max(iy)+1 - nz = np.max(iz)+1 - idlist = (iz + nz*iy + nz*ny*ix).astype(int) - - return idlist - -def writehdf5files(dire): - - if not os.path.exists(dire + "_particles"): - os.makedirs(dire + "_particles") - - eds = emu.EmuDataset(dire) - t = eds.ds.current_time - ad = eds.ds.all_data() - - print(f"Processing {dire} at time {t}") - header = amrex.AMReXParticleHeader(dire+"/neutrinos/Header") - grid_data = GridData(ad) - nlevels = len(header.grids) - print(f"Number of levels: {nlevels}") - assert nlevels==1 - level = 0 - ngrids = len(header.grids[level]) - print(f"Number of grids: {ngrids}") - - # loop over all cells within each grid - for gridID in range(ngrids): - print(f"Processing grid {gridID}") - # read particle data on a single grid - idata, rdata = amrex.read_particle_data(dire, ptype="neutrinos", level_gridID=(level,gridID)) - - # get cell index for each particle - cell_x_index = (rdata[:,rkey['pos_x']] / grid_data.dx).astype(int) - cell_y_index = (rdata[:,rkey['pos_y']] / grid_data.dy).astype(int) - cell_z_index = (rdata[:,rkey['pos_z']] / grid_data.dz).astype(int) - cell_index = np.stack((cell_x_index, cell_y_index, cell_z_index), axis=1) - # get unique cell index - unique_cell_index = np.unique(cell_index, axis=0) - - for i in range(unique_cell_index.shape[0]): - cell_filename = f"{dire}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" - if os.path.exists(cell_filename): - print(f"file {cell_filename} already exists") - else: - mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) - cell_group = rdata[mask_cell_group] - with h5py.File(cell_filename, 'w') as cell_hf: - for label in labels: - cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) - - return dire - -# run the write hdf5 files function in parallel -if __name__ == '__main__': - - dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) - dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" - dir_particles_hdf5 = sorted(glob.glob("plt*_particles")) - dir_particles_hdf5 = [dir_particles_hdf5[i].split('_')[0] for i in range(len(dir_particles_hdf5))] - directories = [d for d in dir_particles_binary if d not in dir_particles_hdf5] - - if not directories: - print("No new directories to process.") - sys.exit(0) - - parser = argparse.ArgumentParser(description='Process some directories.') - parser.add_argument('--index', type=int, default=-2, help='index of the directory to analyze') - args = parser.parse_args() - - if args.index >= -1: - if args.index == -1: - directories = [directories[-1]] - elif args.index < len(directories): - directories = [directories[args.index]] - else: - print(f"Index {args.index} is out of range. Exiting.") - sys.exit(1) - - print(f"Directories to process: {directories}") - - # get NF - eds = emu.EmuDataset(directories[0]) - NF = eds.get_num_flavors() - if NF==2: - rkey, ikey = amrex.get_particle_keys(NF) - labels=['pos_x','pos_y','pos_z', 'time', 'x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N11_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N11_Rebar', 'TrHN', 'Vphase'] - if NF==3: - rkey, ikey = amrex.get_particle_keys(NF) - labels=['pos_x','pos_y','pos_z','time','x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N02_Re', 'N02_Im', 'N11_Re', 'N12_Re', 'N12_Im' ,'N22_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N02_Rebar', 'N02_Imbar', 'N11_Rebar', 'N12_Rebar' ,'N12_Imbar', 'N22_Rebar', 'TrHN', 'Vphase'] - - nproc = mp.cpu_count() - pool = Pool(nproc) - finalresult = pool.map(writehdf5files, directories) \ No newline at end of file diff --git a/Scripts/collisions/write_particles_per_cell_par_in_grid.py b/Scripts/collisions/write_particles_per_cell_par_in_grid.py new file mode 100755 index 0000000..45b3687 --- /dev/null +++ b/Scripts/collisions/write_particles_per_cell_par_in_grid.py @@ -0,0 +1,156 @@ +''' +Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. +This script was based on the script reduced_data.py. It writes all the particle information +clasifyin it per cell. The data can be found in the directories plt*_particles +The particle data is stored in hdf5 files with the following labels: cell_i_j_k.h5 +where i, j, k are the cell indexes in the x, y, z directions respectively. +i=0, j=0, k=0 is the cell with the minimum x, y, z coordinates. +''' + +import os +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/data_reduction') +import numpy as np +import glob +import multiprocessing as mp +import h5py +import amrex_plot_tools as amrex +import emu_yt_module as emu +from multiprocessing import Pool +import argparse +import time +from concurrent.futures import ThreadPoolExecutor + +class GridData(object): + def __init__(self, ad): + x = ad['index','x'].d + y = ad['index','y'].d + z = ad['index','z'].d + dx = ad['index','dx'].d + dy = ad['index','dy'].d + dz = ad['index','dz'].d + self.ad = ad + self.dx = dx[0] + self.dy = dy[0] + self.dz = dz[0] + self.xmin = np.min(x-dx/2.) + self.ymin = np.min(y-dy/2.) + self.zmin = np.min(z-dz/2.) + self.xmax = np.max(x+dx/2.) + self.ymax = np.max(y+dy/2.) + self.zmax = np.max(z+dz/2.) + self.nx = int((self.xmax - self.xmin) / self.dx + 0.5) + self.ny = int((self.ymax - self.ymin) / self.dy + 0.5) + self.nz = int((self.zmax - self.zmin) / self.dz + 0.5) + + def get_particle_cell_ids(self,rdata): + # get coordinates + x = rdata[:,rkey["x"]] + y = rdata[:,rkey["y"]] + z = rdata[:,rkey["z"]] + ix = (x/self.dx).astype(int) + iy = (y/self.dy).astype(int) + iz = (z/self.dz).astype(int) + + # HACK - get this grid's bounds using particle locations + ix -= np.min(ix) + iy -= np.min(iy) + iz -= np.min(iz) + nx = np.max(ix)+1 + ny = np.max(iy)+1 + nz = np.max(iz)+1 + idlist = (iz + nz*iy + nz*ny*ix).astype(int) + + return idlist + +# loop over all cells within each grid +def processing_grid(parms): + + gridID = parms[0] + direct = parms[1] + + print(f"Processing grid {gridID}") + + # read particle data on a single grid + idata, rdata = amrex.read_particle_data(direct, ptype="neutrinos", level_gridID=(level,gridID)) + + # get cell index for each particle + cell_x_index = (rdata[:,rkey['pos_x']] / grid_data.dx).astype(int) + cell_y_index = (rdata[:,rkey['pos_y']] / grid_data.dy).astype(int) + cell_z_index = (rdata[:,rkey['pos_z']] / grid_data.dz).astype(int) + cell_index = np.stack((cell_x_index, cell_y_index, cell_z_index), axis=1) + + # get unique cell index + unique_cell_index = np.unique(cell_index, axis=0) + + def process_cell(i): + cell_filename = f"{direct}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" + if os.path.exists(cell_filename): + print(f"file {cell_filename} already exists") + else: + mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) + cell_group = rdata[mask_cell_group] + with h5py.File(cell_filename, 'w') as cell_hf: + for label in labels: + cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) + + with ThreadPoolExecutor() as executor: + executor.map(process_cell, range(unique_cell_index.shape[0])) + +# Reading directories +dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) +dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" +dir_particles_hdf5 = sorted(glob.glob("plt*_particles")) +dir_particles_hdf5 = [dir_particles_hdf5[i].split('_')[0] for i in range(len(dir_particles_hdf5))] +directories = [d for d in dir_particles_binary if d not in dir_particles_hdf5] + +if not directories: + print("No new directories to process.") + sys.exit(0) + +# get NF +eds = emu.EmuDataset(directories[0]) +NF = eds.get_num_flavors() +if NF==2: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z', 'time', 'x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N11_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N11_Rebar', 'TrHN', 'Vphase'] +if NF==3: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z','time','x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N02_Re', 'N02_Im', 'N11_Re', 'N12_Re', 'N12_Im' ,'N22_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N02_Rebar', 'N02_Imbar', 'N11_Rebar', 'N12_Rebar' ,'N12_Imbar', 'N22_Rebar', 'TrHN', 'Vphase'] + +eds = emu.EmuDataset(directories[0]) +t = eds.ds.current_time +ad = eds.ds.all_data() + +# print(f"Processing {dire} at time {t}") +header = amrex.AMReXParticleHeader(directories[0]+"/neutrinos/Header") +grid_data = GridData(ad) +nlevels = len(header.grids) +assert nlevels==1 +level = 0 +ngrids = len(header.grids[level]) + +parser = argparse.ArgumentParser(description='Process some directories.') +parser.add_argument('--index', type=int, default=-2, help='index of the directory to analyze') +args = parser.parse_args() + +if args.index >= -1: + if args.index == -1: + directories = [directories[-1]] + elif args.index < len(directories): + directories = [directories[args.index]] + else: + print(f"Index {args.index} is out of range. Exiting.") + sys.exit(1) + +print(f"Directories to process: {directories}") + +# Run on CPU parallelized over grids +start_time_gpu = time.time() +for dir in directories: + parameters = [[i, dir] for i in range(ngrids)] + with Pool(mp.cpu_count()) as pool: + pool.map(processing_grid, parameters) +end_time_gpu = time.time() +print(f"GPU processing time: {end_time_gpu - start_time_gpu} seconds") From 2d5a3c6351c9dbbe686443a8b4ef94771f0ef247 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 4 Jan 2025 11:23:46 -0500 Subject: [PATCH 79/83] Solving issue creating particles file --- Scripts/collisions/write_particles_per_cell_par_in_grid.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Scripts/collisions/write_particles_per_cell_par_in_grid.py b/Scripts/collisions/write_particles_per_cell_par_in_grid.py index 45b3687..2f9b5b6 100755 --- a/Scripts/collisions/write_particles_per_cell_par_in_grid.py +++ b/Scripts/collisions/write_particles_per_cell_par_in_grid.py @@ -149,8 +149,14 @@ def process_cell(i): # Run on CPU parallelized over grids start_time_gpu = time.time() for dir in directories: + + if not os.path.exists(dir + "_particles"): + os.makedirs(dir + "_particles") + parameters = [[i, dir] for i in range(ngrids)] + with Pool(mp.cpu_count()) as pool: pool.map(processing_grid, parameters) + end_time_gpu = time.time() print(f"GPU processing time: {end_time_gpu - start_time_gpu} seconds") From 3c62f341f856583b506b0b419f342a5d0d1c9398 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 4 Jan 2025 15:22:43 -0500 Subject: [PATCH 80/83] Delete thread parallelization --- Scripts/collisions/write_particles_per_cell_par_in_grid.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Scripts/collisions/write_particles_per_cell_par_in_grid.py b/Scripts/collisions/write_particles_per_cell_par_in_grid.py index 2f9b5b6..99a4ebf 100755 --- a/Scripts/collisions/write_particles_per_cell_par_in_grid.py +++ b/Scripts/collisions/write_particles_per_cell_par_in_grid.py @@ -84,7 +84,7 @@ def processing_grid(parms): # get unique cell index unique_cell_index = np.unique(cell_index, axis=0) - def process_cell(i): + for i in range(unique_cell_index.shape[0]): cell_filename = f"{direct}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" if os.path.exists(cell_filename): print(f"file {cell_filename} already exists") @@ -95,9 +95,6 @@ def process_cell(i): for label in labels: cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) - with ThreadPoolExecutor() as executor: - executor.map(process_cell, range(unique_cell_index.shape[0])) - # Reading directories dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" From bfbba289203ea14377c419b5313b7c6885370030 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 4 Jan 2025 17:28:37 -0500 Subject: [PATCH 81/83] Update write particles per cell script to recive the grid index as input parameter --- .../collisions/write_particles_per_cell.py | 137 +++++++++--------- 1 file changed, 71 insertions(+), 66 deletions(-) diff --git a/Scripts/collisions/write_particles_per_cell.py b/Scripts/collisions/write_particles_per_cell.py index b5e7868..631d8fc 100755 --- a/Scripts/collisions/write_particles_per_cell.py +++ b/Scripts/collisions/write_particles_per_cell.py @@ -19,6 +19,7 @@ import emu_yt_module as emu from multiprocessing import Pool import argparse +import time class GridData(object): def __init__(self, ad): @@ -77,72 +78,76 @@ def writehdf5files(dire): nlevels = len(header.grids) assert nlevels==1 level = 0 - ngrids = len(header.grids[level]) - # loop over all cells within each grid - for gridID in range(ngrids): - # read particle data on a single grid - idata, rdata = amrex.read_particle_data(dire, ptype="neutrinos", level_gridID=(level,gridID)) - - # get cell index for each particle - cell_x_index = (rdata[:,rkey['pos_x']] / grid_data.dx).astype(int) - cell_y_index = (rdata[:,rkey['pos_y']] / grid_data.dy).astype(int) - cell_z_index = (rdata[:,rkey['pos_z']] / grid_data.dz).astype(int) - cell_index = np.stack((cell_x_index, cell_y_index, cell_z_index), axis=1) - # get unique cell index - unique_cell_index = np.unique(cell_index, axis=0) - - for i in range(unique_cell_index.shape[0]): - cell_filename = f"{dire}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" - if os.path.exists(cell_filename): - print(f"file {cell_filename} already exists") - else: - mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) - cell_group = rdata[mask_cell_group] - with h5py.File(cell_filename, 'w') as cell_hf: - for label in labels: - cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) - - return dire - -# run the write hdf5 files function in parallel -if __name__ == '__main__': - - dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) - dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" - dir_particles_hdf5 = sorted(glob.glob("plt*_particles")) - dir_particles_hdf5 = [dir_particles_hdf5[i].split('_')[0] for i in range(len(dir_particles_hdf5))] - directories = [d for d in dir_particles_binary if d not in dir_particles_hdf5] + gridID = grid_i + grid_data.nx*grid_j + grid_data.nx*grid_data.ny*grid_k + + # read particle data on a single grid + idata, rdata = amrex.read_particle_data(dire, ptype="neutrinos", level_gridID=(level,gridID)) - if not directories: - print("No new directories to process.") - sys.exit(0) - - parser = argparse.ArgumentParser(description='Process some directories.') - parser.add_argument('--index', type=int, default=-2, help='index of the directory to analyze') - args = parser.parse_args() - - if args.index >= -1: - if args.index == -1: - directories = [directories[-1]] - elif args.index < len(directories): - directories = [directories[args.index]] + # get cell index for each particle + cell_x_index = (rdata[:,rkey['pos_x']] / grid_data.dx).astype(int) + cell_y_index = (rdata[:,rkey['pos_y']] / grid_data.dy).astype(int) + cell_z_index = (rdata[:,rkey['pos_z']] / grid_data.dz).astype(int) + cell_index = np.stack((cell_x_index, cell_y_index, cell_z_index), axis=1) + # get unique cell index + unique_cell_index = np.unique(cell_index, axis=0) + + for i in range(unique_cell_index.shape[0]): + cell_filename = f"{dire}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" + if os.path.exists(cell_filename): + print(f"file {cell_filename} already exists") else: - print(f"Index {args.index} is out of range. Exiting.") - sys.exit(1) - - print(f"Directories to process: {directories}") - - # get NF - eds = emu.EmuDataset(directories[0]) - NF = eds.get_num_flavors() - if NF==2: - rkey, ikey = amrex.get_particle_keys(NF) - labels=['pos_x','pos_y','pos_z', 'time', 'x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N11_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N11_Rebar', 'TrHN', 'Vphase'] - if NF==3: - rkey, ikey = amrex.get_particle_keys(NF) - labels=['pos_x','pos_y','pos_z','time','x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N02_Re', 'N02_Im', 'N11_Re', 'N12_Re', 'N12_Im' ,'N22_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N02_Rebar', 'N02_Imbar', 'N11_Rebar', 'N12_Rebar' ,'N12_Imbar', 'N22_Rebar', 'TrHN', 'Vphase'] - - nproc = mp.cpu_count() - pool = Pool(nproc) - finalresult = pool.map(writehdf5files, directories) \ No newline at end of file + mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) + cell_group = rdata[mask_cell_group] + with h5py.File(cell_filename, 'w') as cell_hf: + for label in labels: + cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) + +dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) +dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" +dir_particles_hdf5 = sorted(glob.glob("plt*_particles")) +dir_particles_hdf5 = [dir_particles_hdf5[i].split('_')[0] for i in range(len(dir_particles_hdf5))] +directories = [d for d in dir_particles_binary if d not in dir_particles_hdf5] + +if not directories: + print("No new directories to process.") + sys.exit(0) + +parser = argparse.ArgumentParser(description='Process some directories.') +parser.add_argument('--index', type=int, default=-2, help='index of the directory to analyze') +parser.add_argument('--grid_i', type=int, required=True, help='grid index in the x direction') +parser.add_argument('--grid_j', type=int, required=True, help='grid index in the y direction') +parser.add_argument('--grid_k', type=int, required=True, help='grid index in the z direction') +args = parser.parse_args() + +if args.index >= -1: + if args.index == -1: + directories = [directories[-1]] + elif args.index < len(directories): + directories = [directories[args.index]] + else: + print(f"Index {args.index} is out of range. Exiting.") + sys.exit(1) + +grid_i = args.grid_i +grid_j = args.grid_j +grid_k = args.grid_k + +print(f"Directories to process: {directories}") + +# get NF +eds = emu.EmuDataset(directories[0]) +NF = eds.get_num_flavors() +if NF==2: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z', 'time', 'x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N11_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N11_Rebar', 'TrHN', 'Vphase'] +if NF==3: + rkey, ikey = amrex.get_particle_keys(NF) + labels=['pos_x','pos_y','pos_z','time','x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N02_Re', 'N02_Im', 'N11_Re', 'N12_Re', 'N12_Im' ,'N22_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N02_Rebar', 'N02_Imbar', 'N11_Rebar', 'N12_Rebar' ,'N12_Imbar', 'N22_Rebar', 'TrHN', 'Vphase'] + +start_time_gpu = time.time() +nproc = mp.cpu_count() +pool = Pool(nproc) +pool.map(writehdf5files, directories) +end_time_gpu = time.time() +print(f"Processing time: {end_time_gpu - start_time_gpu} seconds") \ No newline at end of file From 2ab5ab3664a468451ef88222d4a0b0639975cb84 Mon Sep 17 00:00:00 2001 From: erickurquilla1999 Date: Sat, 4 Jan 2025 17:33:06 -0500 Subject: [PATCH 82/83] Delete file --- .../write_particles_per_cell_par_in_grid.py | 159 ------------------ 1 file changed, 159 deletions(-) delete mode 100755 Scripts/collisions/write_particles_per_cell_par_in_grid.py diff --git a/Scripts/collisions/write_particles_per_cell_par_in_grid.py b/Scripts/collisions/write_particles_per_cell_par_in_grid.py deleted file mode 100755 index 99a4ebf..0000000 --- a/Scripts/collisions/write_particles_per_cell_par_in_grid.py +++ /dev/null @@ -1,159 +0,0 @@ -''' -Created by Erick Urquilla, Department of Physics and Astronomy, University of Tennessee, Knoxville. -This script was based on the script reduced_data.py. It writes all the particle information -clasifyin it per cell. The data can be found in the directories plt*_particles -The particle data is stored in hdf5 files with the following labels: cell_i_j_k.h5 -where i, j, k are the cell indexes in the x, y, z directions respectively. -i=0, j=0, k=0 is the cell with the minimum x, y, z coordinates. -''' - -import os -os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' -import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+'/data_reduction') -import numpy as np -import glob -import multiprocessing as mp -import h5py -import amrex_plot_tools as amrex -import emu_yt_module as emu -from multiprocessing import Pool -import argparse -import time -from concurrent.futures import ThreadPoolExecutor - -class GridData(object): - def __init__(self, ad): - x = ad['index','x'].d - y = ad['index','y'].d - z = ad['index','z'].d - dx = ad['index','dx'].d - dy = ad['index','dy'].d - dz = ad['index','dz'].d - self.ad = ad - self.dx = dx[0] - self.dy = dy[0] - self.dz = dz[0] - self.xmin = np.min(x-dx/2.) - self.ymin = np.min(y-dy/2.) - self.zmin = np.min(z-dz/2.) - self.xmax = np.max(x+dx/2.) - self.ymax = np.max(y+dy/2.) - self.zmax = np.max(z+dz/2.) - self.nx = int((self.xmax - self.xmin) / self.dx + 0.5) - self.ny = int((self.ymax - self.ymin) / self.dy + 0.5) - self.nz = int((self.zmax - self.zmin) / self.dz + 0.5) - - def get_particle_cell_ids(self,rdata): - # get coordinates - x = rdata[:,rkey["x"]] - y = rdata[:,rkey["y"]] - z = rdata[:,rkey["z"]] - ix = (x/self.dx).astype(int) - iy = (y/self.dy).astype(int) - iz = (z/self.dz).astype(int) - - # HACK - get this grid's bounds using particle locations - ix -= np.min(ix) - iy -= np.min(iy) - iz -= np.min(iz) - nx = np.max(ix)+1 - ny = np.max(iy)+1 - nz = np.max(iz)+1 - idlist = (iz + nz*iy + nz*ny*ix).astype(int) - - return idlist - -# loop over all cells within each grid -def processing_grid(parms): - - gridID = parms[0] - direct = parms[1] - - print(f"Processing grid {gridID}") - - # read particle data on a single grid - idata, rdata = amrex.read_particle_data(direct, ptype="neutrinos", level_gridID=(level,gridID)) - - # get cell index for each particle - cell_x_index = (rdata[:,rkey['pos_x']] / grid_data.dx).astype(int) - cell_y_index = (rdata[:,rkey['pos_y']] / grid_data.dy).astype(int) - cell_z_index = (rdata[:,rkey['pos_z']] / grid_data.dz).astype(int) - cell_index = np.stack((cell_x_index, cell_y_index, cell_z_index), axis=1) - - # get unique cell index - unique_cell_index = np.unique(cell_index, axis=0) - - for i in range(unique_cell_index.shape[0]): - cell_filename = f"{direct}_particles/cell_{unique_cell_index[i,0]}_{unique_cell_index[i,1]}_{unique_cell_index[i,2]}.h5" - if os.path.exists(cell_filename): - print(f"file {cell_filename} already exists") - else: - mask_cell_group = np.all(cell_index == unique_cell_index[i], axis=1) - cell_group = rdata[mask_cell_group] - with h5py.File(cell_filename, 'w') as cell_hf: - for label in labels: - cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) - -# Reading directories -dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) -dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" -dir_particles_hdf5 = sorted(glob.glob("plt*_particles")) -dir_particles_hdf5 = [dir_particles_hdf5[i].split('_')[0] for i in range(len(dir_particles_hdf5))] -directories = [d for d in dir_particles_binary if d not in dir_particles_hdf5] - -if not directories: - print("No new directories to process.") - sys.exit(0) - -# get NF -eds = emu.EmuDataset(directories[0]) -NF = eds.get_num_flavors() -if NF==2: - rkey, ikey = amrex.get_particle_keys(NF) - labels=['pos_x','pos_y','pos_z', 'time', 'x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N11_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N11_Rebar', 'TrHN', 'Vphase'] -if NF==3: - rkey, ikey = amrex.get_particle_keys(NF) - labels=['pos_x','pos_y','pos_z','time','x', 'y', 'z', 'pupx', 'pupy', 'pupz', 'pupt', 'N00_Re', 'N01_Re', 'N01_Im', 'N02_Re', 'N02_Im', 'N11_Re', 'N12_Re', 'N12_Im' ,'N22_Re', 'N00_Rebar', 'N01_Rebar', 'N01_Imbar', 'N02_Rebar', 'N02_Imbar', 'N11_Rebar', 'N12_Rebar' ,'N12_Imbar', 'N22_Rebar', 'TrHN', 'Vphase'] - -eds = emu.EmuDataset(directories[0]) -t = eds.ds.current_time -ad = eds.ds.all_data() - -# print(f"Processing {dire} at time {t}") -header = amrex.AMReXParticleHeader(directories[0]+"/neutrinos/Header") -grid_data = GridData(ad) -nlevels = len(header.grids) -assert nlevels==1 -level = 0 -ngrids = len(header.grids[level]) - -parser = argparse.ArgumentParser(description='Process some directories.') -parser.add_argument('--index', type=int, default=-2, help='index of the directory to analyze') -args = parser.parse_args() - -if args.index >= -1: - if args.index == -1: - directories = [directories[-1]] - elif args.index < len(directories): - directories = [directories[args.index]] - else: - print(f"Index {args.index} is out of range. Exiting.") - sys.exit(1) - -print(f"Directories to process: {directories}") - -# Run on CPU parallelized over grids -start_time_gpu = time.time() -for dir in directories: - - if not os.path.exists(dir + "_particles"): - os.makedirs(dir + "_particles") - - parameters = [[i, dir] for i in range(ngrids)] - - with Pool(mp.cpu_count()) as pool: - pool.map(processing_grid, parameters) - -end_time_gpu = time.time() -print(f"GPU processing time: {end_time_gpu - start_time_gpu} seconds") From cdd589e2e608bd17ccf2dcf11d84e6881c700cc1 Mon Sep 17 00:00:00 2001 From: Erick Urquilla Date: Sat, 4 Jan 2025 15:30:21 -0800 Subject: [PATCH 83/83] Fix issue with number of amrex grid subdivision --- .../collisions/write_particles_per_cell.py | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/Scripts/collisions/write_particles_per_cell.py b/Scripts/collisions/write_particles_per_cell.py index 631d8fc..254dd85 100755 --- a/Scripts/collisions/write_particles_per_cell.py +++ b/Scripts/collisions/write_particles_per_cell.py @@ -79,7 +79,7 @@ def writehdf5files(dire): assert nlevels==1 level = 0 - gridID = grid_i + grid_data.nx*grid_j + grid_data.nx*grid_data.ny*grid_k + gridID = grid_i + N_grid_x*grid_j + N_grid_x*N_grid_y*grid_k # read particle data on a single grid idata, rdata = amrex.read_particle_data(dire, ptype="neutrinos", level_gridID=(level,gridID)) @@ -103,35 +103,39 @@ def writehdf5files(dire): for label in labels: cell_hf.create_dataset(label, data=cell_group[:, rkey[label]], maxshape=(None,), chunks=True) -dir_particles_binary = sorted(glob.glob("plt*/neutrinos")) -dir_particles_binary = [dir_particles_binary[i].split('/')[0] for i in range(len(dir_particles_binary))] # remove "neutrinos" -dir_particles_hdf5 = sorted(glob.glob("plt*_particles")) -dir_particles_hdf5 = [dir_particles_hdf5[i].split('_')[0] for i in range(len(dir_particles_hdf5))] -directories = [d for d in dir_particles_binary if d not in dir_particles_hdf5] +directories = sorted(glob.glob("plt*/neutrinos")) +directories = [directories[i].split('/')[0] for i in range(len(directories))] # remove "neutrinos" if not directories: print("No new directories to process.") sys.exit(0) parser = argparse.ArgumentParser(description='Process some directories.') -parser.add_argument('--index', type=int, default=-2, help='index of the directory to analyze') -parser.add_argument('--grid_i', type=int, required=True, help='grid index in the x direction') -parser.add_argument('--grid_j', type=int, required=True, help='grid index in the y direction') -parser.add_argument('--grid_k', type=int, required=True, help='grid index in the z direction') +parser.add_argument('--fi', type=int, default=-2, help='index of the directory to analyze') +parser.add_argument('--gi', type=int, required=True, help='grid index in the x direction') +parser.add_argument('--gj', type=int, required=True, help='grid index in the y direction') +parser.add_argument('--gk', type=int, required=True, help='grid index in the z direction') +parser.add_argument('--ngi', type=int, required=True, help='grid index in the x direction') +parser.add_argument('--ngj', type=int, required=True, help='grid index in the y direction') +parser.add_argument('--ngk', type=int, required=True, help='grid index in the z direction') args = parser.parse_args() -if args.index >= -1: - if args.index == -1: +if args.fi >= -1: + if args.fi == -1: directories = [directories[-1]] - elif args.index < len(directories): - directories = [directories[args.index]] + elif args.fi < len(directories): + directories = [directories[args.fi]] else: - print(f"Index {args.index} is out of range. Exiting.") + print(f"Index {args.fi} is out of range. Exiting.") sys.exit(1) -grid_i = args.grid_i -grid_j = args.grid_j -grid_k = args.grid_k +grid_i = args.gi +grid_j = args.gj +grid_k = args.gk + +N_grid_x = args.ngi +N_grid_y = args.ngj +N_grid_z = args.ngk print(f"Directories to process: {directories}")