From 567b0c9e16bb6ca700204ce2f2c71aa13f4eccef Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 14 Nov 2024 10:04:48 -0800 Subject: [PATCH 01/37] region profiling of EB MLMG added test inputs files to setup scaling --- Source/FieldSolver/WarpXSolveFieldsES.cpp | 1 + inputs | 112 ++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 inputs diff --git a/Source/FieldSolver/WarpXSolveFieldsES.cpp b/Source/FieldSolver/WarpXSolveFieldsES.cpp index 6194570cd2d..f6a48761b0c 100644 --- a/Source/FieldSolver/WarpXSolveFieldsES.cpp +++ b/Source/FieldSolver/WarpXSolveFieldsES.cpp @@ -16,6 +16,7 @@ void WarpX::ComputeSpaceChargeField (bool const reset_fields) { WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); + WARPX_PROFILE_REGION("WarpX::ComputeSpaceChargeField()"); using ablastr::fields::Direction; using warpx::fields::FieldType; diff --git a/inputs b/inputs new file mode 100644 index 00000000000..5b7f2344b15 --- /dev/null +++ b/inputs @@ -0,0 +1,112 @@ +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e21 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 30000. # (K) see Chen et al. 2024 +my_constants.Nplasma = 1.e14 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 10 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 + +amr.n_cell = 256 128 # 2*0.1035/256 = 0.0008 m (dx = 1 mm approx.); 0.1054/128 = 0.0008 m (dz = 1 mm approx.)) +amr.max_grid_size = 256 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = absorbing absorbing +boundary.particle_hi = absorbing absorbing + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0508 +my_constants.dx_thick = 0.0015 # dielectric thickness +my_constants.be_xmax = 0.0523 +my_constants.zhi = 0.0127 +my_constants.zlo = -0.0127 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 4 4 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 4 4 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +diagnostics.diags_names = plt chk #plt_eb +plt.diag_type = Full +plt.intervals = 1 +plt.fields_to_plot = phi +plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +chk.diag_type = Full +chk.format = checkpoint +chk.intervals = 1000 +chk.file_min_digits = 8 + From 1b7e5cf1da8fa31d6e5cf62586826f2bacb917e4 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 14 Nov 2024 10:17:09 -0800 Subject: [PATCH 02/37] parameterize inputs for scaling_factor --- inputs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/inputs b/inputs index 5b7f2344b15..a2ba16fc86a 100644 --- a/inputs +++ b/inputs @@ -1,3 +1,8 @@ +# 1 = default size for 1 MPI +# 2 = double physical system size in each direction keeping dx constant; need 4 MPI +# 4 = quad physical system size in each direction keeping dx constant; need 4 MPI, etc. +my_constants.scaling_fac = 1 + # Argon # We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. @@ -20,14 +25,15 @@ warpx.do_electrostatic = labframe warpx.self_fields_required_precision = 1.e-7 # warpx.use_filter = 0 -amr.n_cell = 256 128 # 2*0.1035/256 = 0.0008 m (dx = 1 mm approx.); 0.1054/128 = 0.0008 m (dz = 1 mm approx.)) +# 2*0.1035/256 = 0.0008 m (dx = 1 mm approx.); 0.1054/128 = 0.0008 m (dz = 1 mm approx.)) +amr.n_cell = 256*scaling_fac 128*scaling_fac amr.max_grid_size = 256 -amr.blocking_factor = 8 +amr.blocking_factor = 128 amr.max_level = 0 geometry.dims = 2 -geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) -geometry.prob_hi = 0.1035 0.0527 +geometry.prob_lo = -0.1035*scaling_fac -0.0527*scaling_fac # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035*scaling_fac 0.0527*scaling_fac boundary.field_lo = pec pec boundary.field_hi = pec pec boundary.potential_hi_x = 0. @@ -41,11 +47,11 @@ boundary.particle_hi = absorbing absorbing algo.particle_shape = 1 # EB -my_constants.te_xmax = 0.0508 -my_constants.dx_thick = 0.0015 # dielectric thickness -my_constants.be_xmax = 0.0523 -my_constants.zhi = 0.0127 -my_constants.zlo = -0.0127 +my_constants.te_xmax = 0.0508*scaling_fac +my_constants.dx_thick = 0.0015*scaling_fac # dielectric thickness +my_constants.be_xmax = 0.0523*scaling_fac +my_constants.zhi = 0.0127*scaling_fac +my_constants.zlo = -0.0127*scaling_fac warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " From ada15ede8eb5df97c3c1b43678e3bc40baa12ab2 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 14 Nov 2024 10:25:09 -0800 Subject: [PATCH 03/37] max_grid_size_x and _z --- inputs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inputs b/inputs index a2ba16fc86a..2ea553044fe 100644 --- a/inputs +++ b/inputs @@ -27,7 +27,8 @@ warpx.use_filter = 0 # 2*0.1035/256 = 0.0008 m (dx = 1 mm approx.); 0.1054/128 = 0.0008 m (dz = 1 mm approx.)) amr.n_cell = 256*scaling_fac 128*scaling_fac -amr.max_grid_size = 256 +amr.max_grid_size_x = 256 +amr.max_grid_size_z = 128 amr.blocking_factor = 128 amr.max_level = 0 From a5f9d34ad9ef6beec26e743865ae3abb9f483fe3 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 14 Nov 2024 10:25:50 -0800 Subject: [PATCH 04/37] comment --- inputs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inputs b/inputs index 2ea553044fe..2e600f01b26 100644 --- a/inputs +++ b/inputs @@ -1,6 +1,7 @@ # 1 = default size for 1 MPI # 2 = double physical system size in each direction keeping dx constant; need 4 MPI -# 4 = quad physical system size in each direction keeping dx constant; need 4 MPI, etc. +# 4 = quad physical system size in each direction keeping dx constant; need 16 MPI +# etc. my_constants.scaling_fac = 1 # Argon From e4d61500fba8b8a5a2c89320739ce75cf9f71052 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 14 Nov 2024 10:56:02 -0800 Subject: [PATCH 05/37] scripts, inputs --- inputs => run_plasma/inputs_plasma | 0 run_plasma/run_16gpu.sh | 18 ++++++++++++++++++ run_plasma/run_1gpu.sh | 18 ++++++++++++++++++ run_plasma/run_4gpu.sh | 18 ++++++++++++++++++ 4 files changed, 54 insertions(+) rename inputs => run_plasma/inputs_plasma (100%) create mode 100644 run_plasma/run_16gpu.sh create mode 100644 run_plasma/run_1gpu.sh create mode 100644 run_plasma/run_4gpu.sh diff --git a/inputs b/run_plasma/inputs_plasma similarity index 100% rename from inputs rename to run_plasma/inputs_plasma diff --git a/run_plasma/run_16gpu.sh b/run_plasma/run_16gpu.sh new file mode 100644 index 00000000000..a16e42b8c41 --- /dev/null +++ b/run_plasma/run_16gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 4 +#SBATCH -C gpu +#SBATCH -G 16 +#SBATCH -q debug +#SBATCH -J WarpX_plasma +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 16 -c 32 --cpu_bind=cores -G 16 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs_plasma my_constants.scaling_fac=4 diff --git a/run_plasma/run_1gpu.sh b/run_plasma/run_1gpu.sh new file mode 100644 index 00000000000..2e58bf0a190 --- /dev/null +++ b/run_plasma/run_1gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -C gpu +#SBATCH -G 1 +#SBATCH -q debug +#SBATCH -J WarpX_plasma +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 1 -c 128 --cpu_bind=cores -G 1 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs_plasma my_constants.scaling_fac=1 diff --git a/run_plasma/run_4gpu.sh b/run_plasma/run_4gpu.sh new file mode 100644 index 00000000000..9a408331d3e --- /dev/null +++ b/run_plasma/run_4gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -C gpu +#SBATCH -G 4 +#SBATCH -q debug +#SBATCH -J WarpX_plasma +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 4 -c 32 --cpu_bind=cores -G 4 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs_plasma my_constants.scaling_fac=2 From 256794b2d3bf266341b380e89d79af6490c5a3ec Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 14 Nov 2024 13:34:02 -0800 Subject: [PATCH 06/37] whitespace --- run_plasma/inputs_plasma | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/run_plasma/inputs_plasma b/run_plasma/inputs_plasma index 2e600f01b26..8d25763bdb8 100644 --- a/run_plasma/inputs_plasma +++ b/run_plasma/inputs_plasma @@ -4,8 +4,10 @@ # etc. my_constants.scaling_fac = 1 -# Argon -# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. +amrex.use_gpu_aware_mpi=1 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. my_constants.Ngas = 3.22e21 # 100 mTorr # (m^-3) my_constants.Tgas = 300. # (K) @@ -18,7 +20,7 @@ my_constants.clight = 3.e8 # speed of light in vacuum my_constants.m_e = 9.11e-31 # (kg) my_constants.kb = 1.38e-23 # (J/K) -# amr.restart = ./diags/chk01450000 +# amr.restart = ./diags/chk01450000 max_step = 10 # 2000000 # 5000 RF cycles warpx.verbose = 1 warpx.const_dt = 1.0/(400*freq) @@ -109,7 +111,7 @@ plt.file_min_digits = 8 #plt_eb.diag_type = BoundaryScraping #plt_eb.format = openpmd -#plt_eb.fields_to_plot = phi +#plt_eb.fields_to_plot = phi #plt_eb.particle_field_species = ar_ions #plt_eb.intervals = 190000:200000:1000 From d662dc42f79c5db723601526bc228396fec55181 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 14 Nov 2024 13:42:33 -0800 Subject: [PATCH 07/37] timers --- Source/FieldSolver/WarpXSolveFieldsES.cpp | 7 +++++ run_plasma/inputs_plasma | 32 +++++++++++------------ run_plasma/run_64gpu.sh | 18 +++++++++++++ 3 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 run_plasma/run_64gpu.sh diff --git a/Source/FieldSolver/WarpXSolveFieldsES.cpp b/Source/FieldSolver/WarpXSolveFieldsES.cpp index f6a48761b0c..a5d2992df77 100644 --- a/Source/FieldSolver/WarpXSolveFieldsES.cpp +++ b/Source/FieldSolver/WarpXSolveFieldsES.cpp @@ -17,6 +17,7 @@ void WarpX::ComputeSpaceChargeField (bool const reset_fields) { WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); WARPX_PROFILE_REGION("WarpX::ComputeSpaceChargeField()"); + const auto evolve_time_beg_step = static_cast(amrex::second()); using ablastr::fields::Direction; using warpx::fields::FieldType; @@ -33,4 +34,10 @@ void WarpX::ComputeSpaceChargeField (bool const reset_fields) m_electrostatic_solver->ComputeSpaceChargeField( m_fields, *mypc, myfl.get(), max_level ); + + const auto evolve_time_end_step = static_cast(amrex::second()); + + amrex::Print()<< "Electrostatic time = " + << evolve_time_end_step-evolve_time_beg_step + << " s\n"; } diff --git a/run_plasma/inputs_plasma b/run_plasma/inputs_plasma index 8d25763bdb8..2365d89f2d6 100644 --- a/run_plasma/inputs_plasma +++ b/run_plasma/inputs_plasma @@ -21,7 +21,7 @@ my_constants.m_e = 9.11e-31 # (kg) my_constants.kb = 1.38e-23 # (J/K) # amr.restart = ./diags/chk01450000 -max_step = 10 # 2000000 # 5000 RF cycles +max_step = 20 # 2000000 # 5000 RF cycles warpx.verbose = 1 warpx.const_dt = 1.0/(400*freq) warpx.do_electrostatic = labframe @@ -87,27 +87,27 @@ coll_ion.species = ar_ions coll_ion.background_density = Ngas coll_ion.background_temperature = Tgas coll_ion.scattering_processes = elastic back charge_exchange -coll_ion.elastic_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat -coll_ion.back_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat -coll_ion.charge_exchange_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat +coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat coll_elec.type = background_mcc coll_elec.species = electrons coll_elec.background_density = Ngas coll_elec.background_temperature = Tgas coll_elec.scattering_processes = elastic excitation1 ionization -coll_elec.elastic_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat coll_elec.excitation1_energy = 11.5 -coll_elec.excitation1_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat coll_elec.ionization_energy = 15.7596112 -coll_elec.ionization_cross_section = /home/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat coll_elec.ionization_species = ar_ions -diagnostics.diags_names = plt chk #plt_eb -plt.diag_type = Full -plt.intervals = 1 -plt.fields_to_plot = phi -plt.file_min_digits = 8 +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 #plt_eb.diag_type = BoundaryScraping #plt_eb.format = openpmd @@ -115,8 +115,8 @@ plt.file_min_digits = 8 #plt_eb.particle_field_species = ar_ions #plt_eb.intervals = 190000:200000:1000 -chk.diag_type = Full -chk.format = checkpoint -chk.intervals = 1000 -chk.file_min_digits = 8 +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 diff --git a/run_plasma/run_64gpu.sh b/run_plasma/run_64gpu.sh new file mode 100644 index 00000000000..df05ac14ce8 --- /dev/null +++ b/run_plasma/run_64gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 16 +#SBATCH -C gpu +#SBATCH -G 64 +#SBATCH -q regular +#SBATCH -J WarpX_plasma +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 64 -c 32 --cpu_bind=cores -G 64 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs_plasma my_constants.scaling_fac=8 From 82033f0e414f9e86256a3ba1ea9011ca57c9e682 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Tue, 3 Dec 2024 19:58:25 -0800 Subject: [PATCH 08/37] inputs --- run_plasma/inputs.2d | 115 ++++++++++++++++++++++ run_plasma/{ => old_inputs}/inputs_plasma | 0 2 files changed, 115 insertions(+) create mode 100644 run_plasma/inputs.2d rename run_plasma/{ => old_inputs}/inputs_plasma (100%) diff --git a/run_plasma/inputs.2d b/run_plasma/inputs.2d new file mode 100644 index 00000000000..f648ada5fc3 --- /dev/null +++ b/run_plasma/inputs.2d @@ -0,0 +1,115 @@ +amrex.use_gpu_aware_mpi = 1 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 20 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 + +amr.n_cell = 8800 4480 # 2*0.1035/256 = 0.0008 m (dx = 1 mm approx.); 0.1054/128 = 0.0008 m (dz = 1 mm approx.)) +amr.max_grid_size_x = 4400 +amr.max_grid_size_y = 2240 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = absorbing absorbing +boundary.particle_hi = absorbing absorbing + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 16 16 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 16 16 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + diff --git a/run_plasma/inputs_plasma b/run_plasma/old_inputs/inputs_plasma similarity index 100% rename from run_plasma/inputs_plasma rename to run_plasma/old_inputs/inputs_plasma From 1063c56e35bf7741b58f12f55ae5bc025449af5e Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Tue, 3 Dec 2024 19:58:56 -0800 Subject: [PATCH 09/37] move scripts --- run_plasma/{ => old_inputs}/run_16gpu.sh | 0 run_plasma/{ => old_inputs}/run_1gpu.sh | 0 run_plasma/{ => old_inputs}/run_4gpu.sh | 0 run_plasma/{ => old_inputs}/run_64gpu.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename run_plasma/{ => old_inputs}/run_16gpu.sh (100%) rename run_plasma/{ => old_inputs}/run_1gpu.sh (100%) rename run_plasma/{ => old_inputs}/run_4gpu.sh (100%) rename run_plasma/{ => old_inputs}/run_64gpu.sh (100%) diff --git a/run_plasma/run_16gpu.sh b/run_plasma/old_inputs/run_16gpu.sh similarity index 100% rename from run_plasma/run_16gpu.sh rename to run_plasma/old_inputs/run_16gpu.sh diff --git a/run_plasma/run_1gpu.sh b/run_plasma/old_inputs/run_1gpu.sh similarity index 100% rename from run_plasma/run_1gpu.sh rename to run_plasma/old_inputs/run_1gpu.sh diff --git a/run_plasma/run_4gpu.sh b/run_plasma/old_inputs/run_4gpu.sh similarity index 100% rename from run_plasma/run_4gpu.sh rename to run_plasma/old_inputs/run_4gpu.sh diff --git a/run_plasma/run_64gpu.sh b/run_plasma/old_inputs/run_64gpu.sh similarity index 100% rename from run_plasma/run_64gpu.sh rename to run_plasma/old_inputs/run_64gpu.sh From 064b4727d3f4a156c781897a9b5f0d2ef762d719 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Tue, 3 Dec 2024 19:59:26 -0800 Subject: [PATCH 10/37] scripts --- run_plasma/run_4gpu.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 run_plasma/run_4gpu.sh diff --git a/run_plasma/run_4gpu.sh b/run_plasma/run_4gpu.sh new file mode 100644 index 00000000000..0bc41c644c0 --- /dev/null +++ b/run_plasma/run_4gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -C gpu +#SBATCH -G 4 +#SBATCH -q debug +#SBATCH -J WarpX_plasma +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 4 -c 32 --cpu_bind=cores -G 4 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d From ee2207ca61917af9bdb8d5d6612aff4b5462b2aa Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 5 Dec 2024 11:34:35 -0800 Subject: [PATCH 11/37] switch to 512^2 grids, and 8 8 ppc --- run_plasma/inputs.2d | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/run_plasma/inputs.2d b/run_plasma/inputs.2d index f648ada5fc3..763dfb8ef5d 100644 --- a/run_plasma/inputs.2d +++ b/run_plasma/inputs.2d @@ -22,9 +22,9 @@ warpx.do_electrostatic = labframe warpx.self_fields_required_precision = 1.e-7 # warpx.use_filter = 0 -amr.n_cell = 8800 4480 # 2*0.1035/256 = 0.0008 m (dx = 1 mm approx.); 0.1054/128 = 0.0008 m (dz = 1 mm approx.)) -amr.max_grid_size_x = 4400 -amr.max_grid_size_y = 2240 +amr.n_cell = 4096 2048 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 amr.blocking_factor = 8 amr.max_level = 0 @@ -57,7 +57,7 @@ particles.species_names = electrons ar_ions electrons.species_type = electron electrons.injection_style = nuniformpercell electrons.initialize_self_fields = 0 -electrons.num_particles_per_cell_each_dim = 16 16 +electrons.num_particles_per_cell_each_dim = 8 8 electrons.profile = constant electrons.density = Nplasma electrons.momentum_distribution_type = maxwell_boltzmann @@ -67,7 +67,7 @@ ar_ions.species_type = argon ar_ions.charge = q_e ar_ions.injection_style = nuniformpercell ar_ions.initialize_self_fields = 0 -ar_ions.num_particles_per_cell_each_dim = 16 16 +ar_ions.num_particles_per_cell_each_dim = 8 8 ar_ions.profile = constant ar_ions.density = Nplasma ar_ions.momentum_distribution_type = maxwell_boltzmann @@ -113,3 +113,6 @@ coll_elec.ionization_species = ar_ions #chk.intervals = 1000 #chk.file_min_digits = 8 +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 From ad78875d22f997cf23f8af416f1a6601064019da Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 5 Dec 2024 11:50:41 -0800 Subject: [PATCH 12/37] add sin to eb phi function so it converges --- run_plasma/inputs.2d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_plasma/inputs.2d b/run_plasma/inputs.2d index 763dfb8ef5d..11287261466 100644 --- a/run_plasma/inputs.2d +++ b/run_plasma/inputs.2d @@ -50,7 +50,7 @@ my_constants.be_xmax = 0.052 my_constants.zhi = 0.0128 my_constants.zlo = -0.0128 warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? -warpx.eb_potential(x,y,z,t) = " voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax) " +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " particles.species_names = electrons ar_ions From d34862dc40cb78c09beecd332cae0da2eee902eb Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Thu, 5 Dec 2024 20:21:28 -0800 Subject: [PATCH 13/37] demonstration for how to modify MLMG parameters --- Source/ablastr/fields/PoissonSolver.H | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index d7eeecead1b..412f3433636 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -258,6 +258,9 @@ computePhi ( amrex::LPInfo info; + amrex::Print() << "HERE LPInfo\n"; + amrex::ParmParse pp1("lpinfo"); + for (int lev=0; lev<=finest_level; lev++) { amrex::Array const dx_scaled {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), @@ -397,6 +400,13 @@ computePhi ( mlmg.setMaxIter(max_iters); mlmg.setAlwaysUseBNorm((max_norm_b > 0)); + amrex::Print() << "HERE MLMG\n"; + + amrex::ParmParse pp2("mlmg"); + int xxx; + pp2.query("setPreSmooth",xxx); + mlmg.setPreSmooth(xxx); + const int ng = int(grid_type == utils::enums::GridType::Collocated); // ghost cells if (ng) { // In this case, computeE needs to use ghost nodes data. So we From f35fa9059f86e406a76f1fddf4c025d5c898e9e5 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Fri, 6 Dec 2024 14:16:15 -0800 Subject: [PATCH 14/37] more MLMG options --- Source/ablastr/fields/PoissonSolver.H | 40 +++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 412f3433636..1ffb0709572 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -404,8 +404,44 @@ computePhi ( amrex::ParmParse pp2("mlmg"); int xxx; - pp2.query("setPreSmooth",xxx); - mlmg.setPreSmooth(xxx); + amrex::Real yyy; + if (pp2.query("setPreSmooth",xxx)) { + mlmg.setPreSmooth(xxx); // default 2 + } + if (pp2.query("setPostSmooth",xxx)) { + mlmg.setPostSmooth(xxx); // default 2 + } + if (pp2.query("setFinalSmooth",xxx)) { + mlmg.setFinalSmooth(xxx); // default 8 (when smoother is used as bottom solver) + } + if (pp2.query("setBottomSmooth",xxx)) { + mlmg.setBottomSmooth(xxx); // default 0 + } + if (pp2.query("bottomSolver",xxx)) { + if (xxx == 0) { + mlmg.setBottomSolver(amrex::BottomSolver::Default); // default 0 + } else if (xxx == 1) { + mlmg.setBottomSolver(amrex::BottomSolver::smoother); + } else if (xxx == 2) { + mlmg.setBottomSolver(amrex::BottomSolver::bicgstab); + } else if (xxx == 3) { + mlmg.setBottomSolver(amrex::BottomSolver::cg); + } else if (xxx == 4) { + mlmg.setBottomSolver(amrex::BottomSolver::bicgcg); + } else if (xxx == 5) { + mlmg.setBottomSolver(amrex::BottomSolver::cgbicg); + } else if (xxx == 6) { + mlmg.setBottomSolver(amrex::BottomSolver::hypre); + } else if (xxx == 7) { + mlmg.setBottomSolver(amrex::BottomSolver::petsc); + } + } + if (pp2.query("setBottomTolerance",yyy)) { + mlmg.setBottomTolerance(yyy); // default 1.e-4 + } + if (pp2.query("setBottomToleranceAbs",yyy)) { + mlmg.setBottomToleranceAbs(yyy); // default -1. + } const int ng = int(grid_type == utils::enums::GridType::Collocated); // ghost cells if (ng) { From e3dbd86935c8768c22f23f9b26d6bfc3a3644f31 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Fri, 6 Dec 2024 14:42:15 -0800 Subject: [PATCH 15/37] lpinfo settings --- Source/ablastr/fields/PoissonSolver.H | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 1ffb0709572..f4218081fc7 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -261,6 +261,12 @@ computePhi ( amrex::Print() << "HERE LPInfo\n"; amrex::ParmParse pp1("lpinfo"); + int aaa; + amrex::Real bbb; + if (pp1.query("setMaxCoarseningLevel",aaa)) { + info.setMaxCoarseningLevel(aaa); // default 30 + } + for (int lev=0; lev<=finest_level; lev++) { amrex::Array const dx_scaled {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), From d7c5b60d5ff57bb8e201142204c509f7c79ad145 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Fri, 6 Dec 2024 14:42:41 -0800 Subject: [PATCH 16/37] settings --- run_plasma/inputs.2d | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/run_plasma/inputs.2d b/run_plasma/inputs.2d index 11287261466..7b32320c3d4 100644 --- a/run_plasma/inputs.2d +++ b/run_plasma/inputs.2d @@ -1,5 +1,16 @@ amrex.use_gpu_aware_mpi = 1 +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + # Argon # We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. From 74181ae969441a7e8b9019a134bf55d57c177c3a Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Fri, 6 Dec 2024 14:49:36 -0800 Subject: [PATCH 17/37] 2 gpu test2 gpu test --- run_plasma/run_2gpu.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 run_plasma/run_2gpu.sh diff --git a/run_plasma/run_2gpu.sh b/run_plasma/run_2gpu.sh new file mode 100644 index 00000000000..e8dde725c9c --- /dev/null +++ b/run_plasma/run_2gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -C gpu +#SBATCH -G 2 +#SBATCH -q debug +#SBATCH -J WarpX_plasma +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 2 -c 64 --cpu_bind=cores -G 2 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d From 37b95a72c95f35f664e044c511100a3097806058 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 9 Dec 2024 08:35:55 -0800 Subject: [PATCH 18/37] initial scaling setup --- run_plasma/inputs.2d_2gpu | 129 ++++++++++++++++++++++ run_plasma/{inputs.2d => inputs.2d_32gpu} | 0 run_plasma/inputs.2d_8gpu | 129 ++++++++++++++++++++++ run_plasma/run_2gpu.sh | 2 +- run_plasma/run_32gpu.sh | 18 +++ run_plasma/run_4gpu.sh | 18 --- run_plasma/run_64cpu.sh | 15 +++ run_plasma/run_8gpu.sh | 18 +++ 8 files changed, 310 insertions(+), 19 deletions(-) create mode 100644 run_plasma/inputs.2d_2gpu rename run_plasma/{inputs.2d => inputs.2d_32gpu} (100%) create mode 100644 run_plasma/inputs.2d_8gpu create mode 100644 run_plasma/run_32gpu.sh delete mode 100644 run_plasma/run_4gpu.sh create mode 100644 run_plasma/run_64cpu.sh create mode 100644 run_plasma/run_8gpu.sh diff --git a/run_plasma/inputs.2d_2gpu b/run_plasma/inputs.2d_2gpu new file mode 100644 index 00000000000..44babbef900 --- /dev/null +++ b/run_plasma/inputs.2d_2gpu @@ -0,0 +1,129 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 20 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 + +amr.n_cell = 1024 512 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = absorbing absorbing +boundary.particle_hi = absorbing absorbing + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/inputs.2d b/run_plasma/inputs.2d_32gpu similarity index 100% rename from run_plasma/inputs.2d rename to run_plasma/inputs.2d_32gpu diff --git a/run_plasma/inputs.2d_8gpu b/run_plasma/inputs.2d_8gpu new file mode 100644 index 00000000000..408e81c5469 --- /dev/null +++ b/run_plasma/inputs.2d_8gpu @@ -0,0 +1,129 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 20 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 + +amr.n_cell = 2048 1024 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = absorbing absorbing +boundary.particle_hi = absorbing absorbing + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/run_2gpu.sh b/run_plasma/run_2gpu.sh index e8dde725c9c..a95a3ae8d3f 100644 --- a/run_plasma/run_2gpu.sh +++ b/run_plasma/run_2gpu.sh @@ -15,4 +15,4 @@ export OMP_PROC_BIND=spread # pin to closest NIC to GPU export MPICH_OFI_NIC_POLICY=GPU -srun -n 2 -c 64 --cpu_bind=cores -G 2 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d +srun -n 2 -c 64 --cpu_bind=cores -G 2 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_2gpu diff --git a/run_plasma/run_32gpu.sh b/run_plasma/run_32gpu.sh new file mode 100644 index 00000000000..16c9ee4d06c --- /dev/null +++ b/run_plasma/run_32gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 8 +#SBATCH -C gpu +#SBATCH -G 32 +#SBATCH -q regular +#SBATCH -J WarpX +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +#applications may perform better with --gpu-bind=none instead of --gpu-bind=single:1 +srun -n 32 -c 32 --cpu_bind=cores -G 32 --gpu-bind=single:1 ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_32gpu + diff --git a/run_plasma/run_4gpu.sh b/run_plasma/run_4gpu.sh deleted file mode 100644 index 0bc41c644c0..00000000000 --- a/run_plasma/run_4gpu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -#SBATCH -N 1 -#SBATCH -C gpu -#SBATCH -G 4 -#SBATCH -q debug -#SBATCH -J WarpX_plasma -#SBATCH -t 00:05:00 -#SBATCH -A mp111_g - -#OpenMP settings: -export OMP_NUM_THREADS=1 -export OMP_PLACES=threads -export OMP_PROC_BIND=spread - -# pin to closest NIC to GPU -export MPICH_OFI_NIC_POLICY=GPU - -srun -n 4 -c 32 --cpu_bind=cores -G 4 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d diff --git a/run_plasma/run_64cpu.sh b/run_plasma/run_64cpu.sh new file mode 100644 index 00000000000..5d978b406b4 --- /dev/null +++ b/run_plasma/run_64cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH -C cpu +#SBATCH -q debug +#SBATCH -J WarpX +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 64 -c 4 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_64cpu + diff --git a/run_plasma/run_8gpu.sh b/run_plasma/run_8gpu.sh new file mode 100644 index 00000000000..28b9613ef29 --- /dev/null +++ b/run_plasma/run_8gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 2 +#SBATCH -C gpu +#SBATCH -G 8 +#SBATCH -q regular +#SBATCH -J WarpX +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +#applications may perform better with --gpu-bind=none instead of --gpu-bind=single:1 +srun -n 8 -c 32 --cpu_bind=cores -G 8 --gpu-bind=single:1 ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_8gpu + From b83ae31eb402b00ea02a4e5f8a285b4d5a8eea6b Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 9 Dec 2024 08:38:47 -0800 Subject: [PATCH 19/37] scaling setup --- run_plasma/inputs.2d_128gpu | 129 ++++++++++++++++++++++++++++++++++++ run_plasma/run_128gpu.sh | 18 +++++ run_plasma/run_32gpu.sh | 2 +- run_plasma/run_8gpu.sh | 2 +- 4 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 run_plasma/inputs.2d_128gpu create mode 100644 run_plasma/run_128gpu.sh diff --git a/run_plasma/inputs.2d_128gpu b/run_plasma/inputs.2d_128gpu new file mode 100644 index 00000000000..2a2b87f140b --- /dev/null +++ b/run_plasma/inputs.2d_128gpu @@ -0,0 +1,129 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 20 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 + +amr.n_cell = 8192 4096 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = absorbing absorbing +boundary.particle_hi = absorbing absorbing + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/run_128gpu.sh b/run_plasma/run_128gpu.sh new file mode 100644 index 00000000000..9d252ce29c0 --- /dev/null +++ b/run_plasma/run_128gpu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH -N 32 +#SBATCH -C gpu +#SBATCH -G 128 +#SBATCH -q regular +#SBATCH -J WarpX +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +#applications may perform better with --gpu-bind=none instead of --gpu-bind=single:1 +srun -n 128 -c 32 --cpu_bind=cores -G 128 --gpu-bind=single:1 ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_128gpu + diff --git a/run_plasma/run_32gpu.sh b/run_plasma/run_32gpu.sh index 16c9ee4d06c..b828d04c271 100644 --- a/run_plasma/run_32gpu.sh +++ b/run_plasma/run_32gpu.sh @@ -2,7 +2,7 @@ #SBATCH -N 8 #SBATCH -C gpu #SBATCH -G 32 -#SBATCH -q regular +#SBATCH -q debug #SBATCH -J WarpX #SBATCH -t 00:05:00 #SBATCH -A mp111_g diff --git a/run_plasma/run_8gpu.sh b/run_plasma/run_8gpu.sh index 28b9613ef29..643aa83c7a4 100644 --- a/run_plasma/run_8gpu.sh +++ b/run_plasma/run_8gpu.sh @@ -2,7 +2,7 @@ #SBATCH -N 2 #SBATCH -C gpu #SBATCH -G 8 -#SBATCH -q regular +#SBATCH -q debug #SBATCH -J WarpX #SBATCH -t 00:05:00 #SBATCH -A mp111_g From b6cdf206fad94ce797c41598162e4473453a078d Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 9 Dec 2024 08:40:01 -0800 Subject: [PATCH 20/37] scaling --- run_plasma/run_128gpu.sh | 2 +- run_plasma/run_2gpu.sh | 2 +- run_plasma/run_32gpu.sh | 2 +- run_plasma/run_64cpu.sh | 2 +- run_plasma/run_8gpu.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/run_plasma/run_128gpu.sh b/run_plasma/run_128gpu.sh index 9d252ce29c0..01b7048bd87 100644 --- a/run_plasma/run_128gpu.sh +++ b/run_plasma/run_128gpu.sh @@ -3,7 +3,7 @@ #SBATCH -C gpu #SBATCH -G 128 #SBATCH -q regular -#SBATCH -J WarpX +#SBATCH -J 128gpu #SBATCH -t 00:05:00 #SBATCH -A mp111_g diff --git a/run_plasma/run_2gpu.sh b/run_plasma/run_2gpu.sh index a95a3ae8d3f..4c0d6ce3e15 100644 --- a/run_plasma/run_2gpu.sh +++ b/run_plasma/run_2gpu.sh @@ -3,7 +3,7 @@ #SBATCH -C gpu #SBATCH -G 2 #SBATCH -q debug -#SBATCH -J WarpX_plasma +#SBATCH -J 2gpu #SBATCH -t 00:05:00 #SBATCH -A mp111_g diff --git a/run_plasma/run_32gpu.sh b/run_plasma/run_32gpu.sh index b828d04c271..6b36bcff23b 100644 --- a/run_plasma/run_32gpu.sh +++ b/run_plasma/run_32gpu.sh @@ -3,7 +3,7 @@ #SBATCH -C gpu #SBATCH -G 32 #SBATCH -q debug -#SBATCH -J WarpX +#SBATCH -J 32gpu #SBATCH -t 00:05:00 #SBATCH -A mp111_g diff --git a/run_plasma/run_64cpu.sh b/run_plasma/run_64cpu.sh index 5d978b406b4..cbfe75c15f0 100644 --- a/run_plasma/run_64cpu.sh +++ b/run_plasma/run_64cpu.sh @@ -2,7 +2,7 @@ #SBATCH -N 1 #SBATCH -C cpu #SBATCH -q debug -#SBATCH -J WarpX +#SBATCH -J 64cpu #SBATCH -t 00:05:00 #OpenMP settings: diff --git a/run_plasma/run_8gpu.sh b/run_plasma/run_8gpu.sh index 643aa83c7a4..5d35f2f74b0 100644 --- a/run_plasma/run_8gpu.sh +++ b/run_plasma/run_8gpu.sh @@ -3,7 +3,7 @@ #SBATCH -C gpu #SBATCH -G 8 #SBATCH -q debug -#SBATCH -J WarpX +#SBATCH -J 8gpu #SBATCH -t 00:05:00 #SBATCH -A mp111_g From 259d42b854b8ee5d99df46c55aef4b8e8197ed42 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 9 Dec 2024 08:48:30 -0800 Subject: [PATCH 21/37] fix gpu binding --- run_plasma/run_128gpu.sh | 7 ++++--- run_plasma/run_32gpu.sh | 7 ++++--- run_plasma/run_8gpu.sh | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/run_plasma/run_128gpu.sh b/run_plasma/run_128gpu.sh index 01b7048bd87..d6761f4fa36 100644 --- a/run_plasma/run_128gpu.sh +++ b/run_plasma/run_128gpu.sh @@ -12,7 +12,8 @@ export OMP_NUM_THREADS=1 export OMP_PLACES=threads export OMP_PROC_BIND=spread -#run the application: -#applications may perform better with --gpu-bind=none instead of --gpu-bind=single:1 -srun -n 128 -c 32 --cpu_bind=cores -G 128 --gpu-bind=single:1 ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_128gpu +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 128 -c 32 --cpu_bind=cores -G 128 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_128gpu diff --git a/run_plasma/run_32gpu.sh b/run_plasma/run_32gpu.sh index 6b36bcff23b..466cc8bbcf3 100644 --- a/run_plasma/run_32gpu.sh +++ b/run_plasma/run_32gpu.sh @@ -12,7 +12,8 @@ export OMP_NUM_THREADS=1 export OMP_PLACES=threads export OMP_PROC_BIND=spread -#run the application: -#applications may perform better with --gpu-bind=none instead of --gpu-bind=single:1 -srun -n 32 -c 32 --cpu_bind=cores -G 32 --gpu-bind=single:1 ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_32gpu +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 32 -c 32 --cpu_bind=cores -G 32 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_32gpu diff --git a/run_plasma/run_8gpu.sh b/run_plasma/run_8gpu.sh index 5d35f2f74b0..fa3eed7d8a7 100644 --- a/run_plasma/run_8gpu.sh +++ b/run_plasma/run_8gpu.sh @@ -12,7 +12,8 @@ export OMP_NUM_THREADS=1 export OMP_PLACES=threads export OMP_PROC_BIND=spread -#run the application: -#applications may perform better with --gpu-bind=none instead of --gpu-bind=single:1 -srun -n 8 -c 32 --cpu_bind=cores -G 8 --gpu-bind=single:1 ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_8gpu +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 8 -c 32 --cpu_bind=cores -G 8 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_8gpu From 634818a205c41aa2a6b6d398607edcd60e9f79d4 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 9 Dec 2024 09:55:06 -0800 Subject: [PATCH 22/37] updated scaling tests --- run_plasma/inputs.2d_128gpu | 9 +++++---- run_plasma/inputs.2d_2gpu | 9 +++++---- run_plasma/inputs.2d_32gpu | 9 +++++---- run_plasma/inputs.2d_8gpu | 9 +++++---- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/run_plasma/inputs.2d_128gpu b/run_plasma/inputs.2d_128gpu index 2a2b87f140b..9026c21ef11 100644 --- a/run_plasma/inputs.2d_128gpu +++ b/run_plasma/inputs.2d_128gpu @@ -26,12 +26,13 @@ my_constants.m_e = 9.11e-31 # (kg) my_constants.kb = 1.38e-23 # (J/K) # amr.restart = ./diags/chk01450000 -max_step = 20 # 2000000 # 5000 RF cycles +max_step = 100 # 2000000 # 5000 RF cycles warpx.verbose = 1 warpx.const_dt = 1.0/(400*freq) warpx.do_electrostatic = labframe warpx.self_fields_required_precision = 1.e-7 # warpx.use_filter = 0 +warpx.sort_intervals = -1 amr.n_cell = 8192 4096 amr.max_grid_size_x = 512 @@ -48,8 +49,8 @@ boundary.potential_hi_x = 0. boundary.potential_lo_z = 0. boundary.potential_lo_x = 0. boundary.potential_hi_z = 0. -boundary.particle_lo = absorbing absorbing -boundary.particle_hi = absorbing absorbing +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting # Order of particle shape factors algo.particle_shape = 1 @@ -85,7 +86,7 @@ ar_ions.momentum_distribution_type = maxwell_boltzmann ar_ions.theta = (kb*Tgas/(Mion*clight^2)) ar_ions.save_particles_at_eb = 1 -collisions.collision_names = coll_elec coll_ion +#collisions.collision_names = coll_elec coll_ion coll_ion.type = background_mcc coll_ion.species = ar_ions coll_ion.background_density = Ngas diff --git a/run_plasma/inputs.2d_2gpu b/run_plasma/inputs.2d_2gpu index 44babbef900..72a44f82a9c 100644 --- a/run_plasma/inputs.2d_2gpu +++ b/run_plasma/inputs.2d_2gpu @@ -26,12 +26,13 @@ my_constants.m_e = 9.11e-31 # (kg) my_constants.kb = 1.38e-23 # (J/K) # amr.restart = ./diags/chk01450000 -max_step = 20 # 2000000 # 5000 RF cycles +max_step = 100 # 2000000 # 5000 RF cycles warpx.verbose = 1 warpx.const_dt = 1.0/(400*freq) warpx.do_electrostatic = labframe warpx.self_fields_required_precision = 1.e-7 # warpx.use_filter = 0 +warpx.sort_intervals = -1 amr.n_cell = 1024 512 amr.max_grid_size_x = 512 @@ -48,8 +49,8 @@ boundary.potential_hi_x = 0. boundary.potential_lo_z = 0. boundary.potential_lo_x = 0. boundary.potential_hi_z = 0. -boundary.particle_lo = absorbing absorbing -boundary.particle_hi = absorbing absorbing +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting # Order of particle shape factors algo.particle_shape = 1 @@ -85,7 +86,7 @@ ar_ions.momentum_distribution_type = maxwell_boltzmann ar_ions.theta = (kb*Tgas/(Mion*clight^2)) ar_ions.save_particles_at_eb = 1 -collisions.collision_names = coll_elec coll_ion +#collisions.collision_names = coll_elec coll_ion coll_ion.type = background_mcc coll_ion.species = ar_ions coll_ion.background_density = Ngas diff --git a/run_plasma/inputs.2d_32gpu b/run_plasma/inputs.2d_32gpu index 7b32320c3d4..1f12990b980 100644 --- a/run_plasma/inputs.2d_32gpu +++ b/run_plasma/inputs.2d_32gpu @@ -26,12 +26,13 @@ my_constants.m_e = 9.11e-31 # (kg) my_constants.kb = 1.38e-23 # (J/K) # amr.restart = ./diags/chk01450000 -max_step = 20 # 2000000 # 5000 RF cycles +max_step = 100 # 2000000 # 5000 RF cycles warpx.verbose = 1 warpx.const_dt = 1.0/(400*freq) warpx.do_electrostatic = labframe warpx.self_fields_required_precision = 1.e-7 # warpx.use_filter = 0 +warpx.sort_intervals = -1 amr.n_cell = 4096 2048 amr.max_grid_size_x = 512 @@ -48,8 +49,8 @@ boundary.potential_hi_x = 0. boundary.potential_lo_z = 0. boundary.potential_lo_x = 0. boundary.potential_hi_z = 0. -boundary.particle_lo = absorbing absorbing -boundary.particle_hi = absorbing absorbing +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting # Order of particle shape factors algo.particle_shape = 1 @@ -85,7 +86,7 @@ ar_ions.momentum_distribution_type = maxwell_boltzmann ar_ions.theta = (kb*Tgas/(Mion*clight^2)) ar_ions.save_particles_at_eb = 1 -collisions.collision_names = coll_elec coll_ion +#collisions.collision_names = coll_elec coll_ion coll_ion.type = background_mcc coll_ion.species = ar_ions coll_ion.background_density = Ngas diff --git a/run_plasma/inputs.2d_8gpu b/run_plasma/inputs.2d_8gpu index 408e81c5469..012c8cedd5f 100644 --- a/run_plasma/inputs.2d_8gpu +++ b/run_plasma/inputs.2d_8gpu @@ -26,12 +26,13 @@ my_constants.m_e = 9.11e-31 # (kg) my_constants.kb = 1.38e-23 # (J/K) # amr.restart = ./diags/chk01450000 -max_step = 20 # 2000000 # 5000 RF cycles +max_step = 100 # 2000000 # 5000 RF cycles warpx.verbose = 1 warpx.const_dt = 1.0/(400*freq) warpx.do_electrostatic = labframe warpx.self_fields_required_precision = 1.e-7 # warpx.use_filter = 0 +warpx.sort_intervals = -1 amr.n_cell = 2048 1024 amr.max_grid_size_x = 512 @@ -48,8 +49,8 @@ boundary.potential_hi_x = 0. boundary.potential_lo_z = 0. boundary.potential_lo_x = 0. boundary.potential_hi_z = 0. -boundary.particle_lo = absorbing absorbing -boundary.particle_hi = absorbing absorbing +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting # Order of particle shape factors algo.particle_shape = 1 @@ -85,7 +86,7 @@ ar_ions.momentum_distribution_type = maxwell_boltzmann ar_ions.theta = (kb*Tgas/(Mion*clight^2)) ar_ions.save_particles_at_eb = 1 -collisions.collision_names = coll_elec coll_ion +#collisions.collision_names = coll_elec coll_ion coll_ion.type = background_mcc coll_ion.species = ar_ions coll_ion.background_density = Ngas From ac889557b068af2cdd8380fb011953fbf5191c1d Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 9 Dec 2024 10:03:02 -0800 Subject: [PATCH 23/37] cpu --- run_plasma/inputs.2d_64cpu | 130 +++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 run_plasma/inputs.2d_64cpu diff --git a/run_plasma/inputs.2d_64cpu b/run_plasma/inputs.2d_64cpu new file mode 100644 index 00000000000..5eefd1e0bc6 --- /dev/null +++ b/run_plasma/inputs.2d_64cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 1024 512 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 From d2e8410d1541c8d795adf7e307e904fb13947993 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Tue, 10 Dec 2024 15:01:46 -0800 Subject: [PATCH 24/37] timers --- Source/FieldSolver/WarpXSolveFieldsES.cpp | 8 +------- Source/ablastr/fields/PoissonSolver.H | 11 ++++++++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Source/FieldSolver/WarpXSolveFieldsES.cpp b/Source/FieldSolver/WarpXSolveFieldsES.cpp index a5d2992df77..169e9f5fc49 100644 --- a/Source/FieldSolver/WarpXSolveFieldsES.cpp +++ b/Source/FieldSolver/WarpXSolveFieldsES.cpp @@ -17,7 +17,7 @@ void WarpX::ComputeSpaceChargeField (bool const reset_fields) { WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); WARPX_PROFILE_REGION("WarpX::ComputeSpaceChargeField()"); - const auto evolve_time_beg_step = static_cast(amrex::second()); + using ablastr::fields::Direction; using warpx::fields::FieldType; @@ -34,10 +34,4 @@ void WarpX::ComputeSpaceChargeField (bool const reset_fields) m_electrostatic_solver->ComputeSpaceChargeField( m_fields, *mypc, myfl.get(), max_level ); - - const auto evolve_time_end_step = static_cast(amrex::second()); - - amrex::Print()<< "Electrostatic time = " - << evolve_time_end_step-evolve_time_beg_step - << " s\n"; } diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index f4218081fc7..17c19764467 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -456,11 +456,20 @@ computePhi ( mlmg.setFinalFillBC(true); } + + const auto mlmg_time_beg_step = static_cast(amrex::second()); + // Solve Poisson equation at lev mlmg.solve( {phi[lev]}, {rho[lev]}, relative_tolerance, absolute_tolerance ); - const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; + const auto mlmg_time_end_step = static_cast(amrex::second()); + + amrex::Print()<< "MLMG time = " + << mlmg_time_end_step-mlmg_time_beg_step + << " s\n"; + + const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; const int ncomp = linop->getNComp(); // needed for solving the levels by levels: From 31bfcdae6729d372d06bb9c3fa8772224065be19 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Tue, 10 Dec 2024 16:48:20 -0800 Subject: [PATCH 25/37] MLLinop settings --- Source/ablastr/fields/PoissonSolver.H | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 17c19764467..bb6a71e7fc4 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -262,7 +262,24 @@ computePhi ( amrex::ParmParse pp1("lpinfo"); int aaa; - amrex::Real bbb; + if (pp1.query("do_agglomeration",aaa)) { + info.setAgglomeration(aaa); // default true + } + if (pp1.query("agg_grid_size",aaa)) { + info.setAgglomerationGridSize(aaa); // default -1 (architecture-dependent setting) + } + if (pp1.query("do_consolidation",aaa)) { + info.setConsolidation(aaa); // default true + } + if (pp1.query("con_grid_size",aaa)) { + info.setConsolidationGridSize(aaa); // default -1 (architecture-dependent setting) + } + if (pp1.query("con_ratio",aaa)) { + info.setConsolidationRatio(aaa); // default 2 + } + if (pp1.query("con_strategy",aaa)) { + info.setConsolidationStrategy(aaa); // default 3 + } if (pp1.query("setMaxCoarseningLevel",aaa)) { info.setMaxCoarseningLevel(aaa); // default 30 } From 7f07527065ecc7705082ce2ade5ba9910a59f8c6 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Tue, 10 Dec 2024 16:50:51 -0800 Subject: [PATCH 26/37] remove old stuff --- run_plasma/old_inputs/inputs_plasma | 122 ---------------------------- run_plasma/old_inputs/run_16gpu.sh | 18 ---- run_plasma/old_inputs/run_1gpu.sh | 18 ---- run_plasma/old_inputs/run_4gpu.sh | 18 ---- run_plasma/old_inputs/run_64gpu.sh | 18 ---- 5 files changed, 194 deletions(-) delete mode 100644 run_plasma/old_inputs/inputs_plasma delete mode 100644 run_plasma/old_inputs/run_16gpu.sh delete mode 100644 run_plasma/old_inputs/run_1gpu.sh delete mode 100644 run_plasma/old_inputs/run_4gpu.sh delete mode 100644 run_plasma/old_inputs/run_64gpu.sh diff --git a/run_plasma/old_inputs/inputs_plasma b/run_plasma/old_inputs/inputs_plasma deleted file mode 100644 index 2365d89f2d6..00000000000 --- a/run_plasma/old_inputs/inputs_plasma +++ /dev/null @@ -1,122 +0,0 @@ -# 1 = default size for 1 MPI -# 2 = double physical system size in each direction keeping dx constant; need 4 MPI -# 4 = quad physical system size in each direction keeping dx constant; need 16 MPI -# etc. -my_constants.scaling_fac = 1 - -amrex.use_gpu_aware_mpi=1 - -# Argon -# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. - -my_constants.Ngas = 3.22e21 # 100 mTorr # (m^-3) -my_constants.Tgas = 300. # (K) -my_constants.Te = 30000. # (K) see Chen et al. 2024 -my_constants.Nplasma = 1.e14 # (m^-3) see Chen et al. 2024 -my_constants.freq = 13.56e6 # (Hz) -my_constants.Mion = 6.63e-26 # (kg) -my_constants.voltage = 100. # (V) -my_constants.clight = 3.e8 # speed of light in vacuum -my_constants.m_e = 9.11e-31 # (kg) -my_constants.kb = 1.38e-23 # (J/K) - -# amr.restart = ./diags/chk01450000 -max_step = 20 # 2000000 # 5000 RF cycles -warpx.verbose = 1 -warpx.const_dt = 1.0/(400*freq) -warpx.do_electrostatic = labframe -warpx.self_fields_required_precision = 1.e-7 # -warpx.use_filter = 0 - -# 2*0.1035/256 = 0.0008 m (dx = 1 mm approx.); 0.1054/128 = 0.0008 m (dz = 1 mm approx.)) -amr.n_cell = 256*scaling_fac 128*scaling_fac -amr.max_grid_size_x = 256 -amr.max_grid_size_z = 128 -amr.blocking_factor = 128 -amr.max_level = 0 - -geometry.dims = 2 -geometry.prob_lo = -0.1035*scaling_fac -0.0527*scaling_fac # x z # cover complete chamber (do not exploit symmetry) -geometry.prob_hi = 0.1035*scaling_fac 0.0527*scaling_fac -boundary.field_lo = pec pec -boundary.field_hi = pec pec -boundary.potential_hi_x = 0. -boundary.potential_lo_z = 0. -boundary.potential_lo_x = 0. -boundary.potential_hi_z = 0. -boundary.particle_lo = absorbing absorbing -boundary.particle_hi = absorbing absorbing - -# Order of particle shape factors -algo.particle_shape = 1 - -# EB -my_constants.te_xmax = 0.0508*scaling_fac -my_constants.dx_thick = 0.0015*scaling_fac # dielectric thickness -my_constants.be_xmax = 0.0523*scaling_fac -my_constants.zhi = 0.0127*scaling_fac -my_constants.zlo = -0.0127*scaling_fac -warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? -warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " - -particles.species_names = electrons ar_ions - -electrons.species_type = electron -electrons.injection_style = nuniformpercell -electrons.initialize_self_fields = 0 -electrons.num_particles_per_cell_each_dim = 4 4 -electrons.profile = constant -electrons.density = Nplasma -electrons.momentum_distribution_type = maxwell_boltzmann -electrons.theta = (kb*Te/(m_e*clight^2)) - -ar_ions.species_type = argon -ar_ions.charge = q_e -ar_ions.injection_style = nuniformpercell -ar_ions.initialize_self_fields = 0 -ar_ions.num_particles_per_cell_each_dim = 4 4 -ar_ions.profile = constant -ar_ions.density = Nplasma -ar_ions.momentum_distribution_type = maxwell_boltzmann -ar_ions.theta = (kb*Tgas/(Mion*clight^2)) -ar_ions.save_particles_at_eb = 1 - -collisions.collision_names = coll_elec coll_ion -coll_ion.type = background_mcc -coll_ion.species = ar_ions -coll_ion.background_density = Ngas -coll_ion.background_temperature = Tgas -coll_ion.scattering_processes = elastic back charge_exchange -coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat -coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat -coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat - -coll_elec.type = background_mcc -coll_elec.species = electrons -coll_elec.background_density = Ngas -coll_elec.background_temperature = Tgas -coll_elec.scattering_processes = elastic excitation1 ionization -coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat -coll_elec.excitation1_energy = 11.5 -coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat -coll_elec.ionization_energy = 15.7596112 -coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat -coll_elec.ionization_species = ar_ions - -#diagnostics.diags_names = plt chk #plt_eb -#plt.diag_type = Full -#plt.intervals = 1 -#plt.fields_to_plot = phi -#plt.file_min_digits = 8 - -#plt_eb.diag_type = BoundaryScraping -#plt_eb.format = openpmd -#plt_eb.fields_to_plot = phi -#plt_eb.particle_field_species = ar_ions -#plt_eb.intervals = 190000:200000:1000 - -#chk.diag_type = Full -#chk.format = checkpoint -#chk.intervals = 1000 -#chk.file_min_digits = 8 - diff --git a/run_plasma/old_inputs/run_16gpu.sh b/run_plasma/old_inputs/run_16gpu.sh deleted file mode 100644 index a16e42b8c41..00000000000 --- a/run_plasma/old_inputs/run_16gpu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -#SBATCH -N 4 -#SBATCH -C gpu -#SBATCH -G 16 -#SBATCH -q debug -#SBATCH -J WarpX_plasma -#SBATCH -t 00:05:00 -#SBATCH -A mp111_g - -#OpenMP settings: -export OMP_NUM_THREADS=1 -export OMP_PLACES=threads -export OMP_PROC_BIND=spread - -# pin to closest NIC to GPU -export MPICH_OFI_NIC_POLICY=GPU - -srun -n 16 -c 32 --cpu_bind=cores -G 16 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs_plasma my_constants.scaling_fac=4 diff --git a/run_plasma/old_inputs/run_1gpu.sh b/run_plasma/old_inputs/run_1gpu.sh deleted file mode 100644 index 2e58bf0a190..00000000000 --- a/run_plasma/old_inputs/run_1gpu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -#SBATCH -N 1 -#SBATCH -C gpu -#SBATCH -G 1 -#SBATCH -q debug -#SBATCH -J WarpX_plasma -#SBATCH -t 00:05:00 -#SBATCH -A mp111_g - -#OpenMP settings: -export OMP_NUM_THREADS=1 -export OMP_PLACES=threads -export OMP_PROC_BIND=spread - -# pin to closest NIC to GPU -export MPICH_OFI_NIC_POLICY=GPU - -srun -n 1 -c 128 --cpu_bind=cores -G 1 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs_plasma my_constants.scaling_fac=1 diff --git a/run_plasma/old_inputs/run_4gpu.sh b/run_plasma/old_inputs/run_4gpu.sh deleted file mode 100644 index 9a408331d3e..00000000000 --- a/run_plasma/old_inputs/run_4gpu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -#SBATCH -N 1 -#SBATCH -C gpu -#SBATCH -G 4 -#SBATCH -q debug -#SBATCH -J WarpX_plasma -#SBATCH -t 00:05:00 -#SBATCH -A mp111_g - -#OpenMP settings: -export OMP_NUM_THREADS=1 -export OMP_PLACES=threads -export OMP_PROC_BIND=spread - -# pin to closest NIC to GPU -export MPICH_OFI_NIC_POLICY=GPU - -srun -n 4 -c 32 --cpu_bind=cores -G 4 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs_plasma my_constants.scaling_fac=2 diff --git a/run_plasma/old_inputs/run_64gpu.sh b/run_plasma/old_inputs/run_64gpu.sh deleted file mode 100644 index df05ac14ce8..00000000000 --- a/run_plasma/old_inputs/run_64gpu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -#SBATCH -N 16 -#SBATCH -C gpu -#SBATCH -G 64 -#SBATCH -q regular -#SBATCH -J WarpX_plasma -#SBATCH -t 00:05:00 -#SBATCH -A mp111_g - -#OpenMP settings: -export OMP_NUM_THREADS=1 -export OMP_PLACES=threads -export OMP_PROC_BIND=spread - -# pin to closest NIC to GPU -export MPICH_OFI_NIC_POLICY=GPU - -srun -n 64 -c 32 --cpu_bind=cores -G 64 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs_plasma my_constants.scaling_fac=8 From 76645c4bf2b73c55804406b124390d4916693812 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Wed, 11 Dec 2024 13:21:32 -0800 Subject: [PATCH 27/37] mp111_g --- .../perlmutter-nersc/perlmutter_gpu_warpx.profile.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 1e5325e29b9..c08db3287ea 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -1,5 +1,5 @@ # please set your project account -export proj="" # change me! GPU projects must end in "..._g" +export proj="mp111_g" # change me! GPU projects must end in "..._g" # remembers the location of this script export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) From ae8cb996f0c89f06c161b5d79d18a441aafb2961 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Wed, 11 Dec 2024 13:27:03 -0800 Subject: [PATCH 28/37] point to global warpx-data --- run_plasma/inputs.2d_128gpu | 12 ++++++------ run_plasma/inputs.2d_2gpu | 12 ++++++------ run_plasma/inputs.2d_32gpu | 12 ++++++------ run_plasma/inputs.2d_64cpu | 12 ++++++------ run_plasma/inputs.2d_8gpu | 12 ++++++------ 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/run_plasma/inputs.2d_128gpu b/run_plasma/inputs.2d_128gpu index 9026c21ef11..9cb8f6100d0 100644 --- a/run_plasma/inputs.2d_128gpu +++ b/run_plasma/inputs.2d_128gpu @@ -92,20 +92,20 @@ coll_ion.species = ar_ions coll_ion.background_density = Ngas coll_ion.background_temperature = Tgas coll_ion.scattering_processes = elastic back charge_exchange -coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat -coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat -coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat coll_elec.type = background_mcc coll_elec.species = electrons coll_elec.background_density = Ngas coll_elec.background_temperature = Tgas coll_elec.scattering_processes = elastic excitation1 ionization -coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat coll_elec.excitation1_energy = 11.5 -coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat coll_elec.ionization_energy = 15.7596112 -coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat coll_elec.ionization_species = ar_ions #diagnostics.diags_names = plt chk #plt_eb diff --git a/run_plasma/inputs.2d_2gpu b/run_plasma/inputs.2d_2gpu index 72a44f82a9c..4f342cded91 100644 --- a/run_plasma/inputs.2d_2gpu +++ b/run_plasma/inputs.2d_2gpu @@ -92,20 +92,20 @@ coll_ion.species = ar_ions coll_ion.background_density = Ngas coll_ion.background_temperature = Tgas coll_ion.scattering_processes = elastic back charge_exchange -coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat -coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat -coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat coll_elec.type = background_mcc coll_elec.species = electrons coll_elec.background_density = Ngas coll_elec.background_temperature = Tgas coll_elec.scattering_processes = elastic excitation1 ionization -coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat coll_elec.excitation1_energy = 11.5 -coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat coll_elec.ionization_energy = 15.7596112 -coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat coll_elec.ionization_species = ar_ions #diagnostics.diags_names = plt chk #plt_eb diff --git a/run_plasma/inputs.2d_32gpu b/run_plasma/inputs.2d_32gpu index 1f12990b980..37ae7e71995 100644 --- a/run_plasma/inputs.2d_32gpu +++ b/run_plasma/inputs.2d_32gpu @@ -92,20 +92,20 @@ coll_ion.species = ar_ions coll_ion.background_density = Ngas coll_ion.background_temperature = Tgas coll_ion.scattering_processes = elastic back charge_exchange -coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat -coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat -coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat coll_elec.type = background_mcc coll_elec.species = electrons coll_elec.background_density = Ngas coll_elec.background_temperature = Tgas coll_elec.scattering_processes = elastic excitation1 ionization -coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat coll_elec.excitation1_energy = 11.5 -coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat coll_elec.ionization_energy = 15.7596112 -coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat coll_elec.ionization_species = ar_ions #diagnostics.diags_names = plt chk #plt_eb diff --git a/run_plasma/inputs.2d_64cpu b/run_plasma/inputs.2d_64cpu index 5eefd1e0bc6..98cfbd8af90 100644 --- a/run_plasma/inputs.2d_64cpu +++ b/run_plasma/inputs.2d_64cpu @@ -92,20 +92,20 @@ coll_ion.species = ar_ions coll_ion.background_density = Ngas coll_ion.background_temperature = Tgas coll_ion.scattering_processes = elastic back charge_exchange -coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat -coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat -coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat coll_elec.type = background_mcc coll_elec.species = electrons coll_elec.background_density = Ngas coll_elec.background_temperature = Tgas coll_elec.scattering_processes = elastic excitation1 ionization -coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat coll_elec.excitation1_energy = 11.5 -coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat coll_elec.ionization_energy = 15.7596112 -coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat coll_elec.ionization_species = ar_ions #diagnostics.diags_names = plt chk #plt_eb diff --git a/run_plasma/inputs.2d_8gpu b/run_plasma/inputs.2d_8gpu index 012c8cedd5f..558e9ee7b54 100644 --- a/run_plasma/inputs.2d_8gpu +++ b/run_plasma/inputs.2d_8gpu @@ -92,20 +92,20 @@ coll_ion.species = ar_ions coll_ion.background_density = Ngas coll_ion.background_temperature = Tgas coll_ion.scattering_processes = elastic back charge_exchange -coll_ion.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat -coll_ion.back_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat -coll_ion.charge_exchange_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat coll_elec.type = background_mcc coll_elec.species = electrons coll_elec.background_density = Ngas coll_elec.background_temperature = Tgas coll_elec.scattering_processes = elastic excitation1 ionization -coll_elec.elastic_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat coll_elec.excitation1_energy = 11.5 -coll_elec.excitation1_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat coll_elec.ionization_energy = 15.7596112 -coll_elec.ionization_cross_section = /global/homes/a/ajnonaka/CCSEPackages/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat coll_elec.ionization_species = ar_ions #diagnostics.diags_names = plt chk #plt_eb From 1ec8d89fc795a3f5bd32f26dc7713b35e0553e6b Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 16 Dec 2024 09:54:52 -0800 Subject: [PATCH 29/37] attempt at GMRES --- Source/ablastr/fields/PoissonSolver.H | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 5d04c5a5c86..71781a00fd5 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -51,6 +51,7 @@ #ifdef AMREX_USE_EB # include #endif +#include #include #include @@ -428,6 +429,10 @@ computePhi ( amrex::ParmParse pp2("mlmg"); int xxx; amrex::Real yyy; + + int solver_type = 0; // 0 = MLMG, 1 = GMRES + pp2.query("solver_type",solver_type); + if (pp2.query("setPreSmooth",xxx)) { mlmg.setPreSmooth(xxx); // default 2 } @@ -473,12 +478,16 @@ computePhi ( mlmg.setFinalFillBC(true); } - const auto mlmg_time_beg_step = static_cast(amrex::second()); + + amrex::GMRESMLMG gmsolve(mlmg); // Solve Poisson equation at lev - mlmg.solve( {phi[lev]}, {rho[lev]}, - relative_tolerance, absolute_tolerance ); + if (solver_type == 0) { + mlmg.solve( {phi[lev]}, {rho[lev]}, relative_tolerance, absolute_tolerance ); + } else { + gmsolve.solve( *phi[lev], *rho[lev], relative_tolerance, absolute_tolerance ); + } const auto mlmg_time_end_step = static_cast(amrex::second()); From 6f94e1ca34478785663675c18fe57df9e8f55fe3 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 16 Dec 2024 16:55:07 -0800 Subject: [PATCH 30/37] attempt at getting gradient of phi --- Source/ablastr/fields/PoissonSolver.H | 39 ++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 71781a00fd5..e8ba35743cc 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -489,6 +489,34 @@ computePhi ( gmsolve.solve( *phi[lev], *rho[lev], relative_tolerance, absolute_tolerance ); } + auto & warpx = WarpX::GetInstance(); + // EB: use AMReX to directly calculate the electric field since with EB's the + // simple finite difference scheme in WarpX::computeE sometimes fails + + // TODO: maybe make this a helper function or pass Efield_fp directly + amrex::Vector< amrex::Array > e_field; + e_field.push_back( +#if defined(WARPX_DIM_1D_Z) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) + } +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, 0), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) + } +#elif defined(WARPX_DIM_3D) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, 0), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{1}, 0), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) + } +#endif + ); + + + mlmg.getFluxes(e_field, phi); + const auto mlmg_time_end_step = static_cast(amrex::second()); amrex::Print()<< "MLMG time = " @@ -513,12 +541,15 @@ computePhi ( ng); } - // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same_v) { - if (post_phi_calculation.has_value()) { - post_phi_calculation.value()(mlmg, lev); + if (solver_type == 0) { + // Run additional operations, such as calculation of the E field for embedded boundaries + if constexpr (!std::is_same_v) { + if (post_phi_calculation.has_value()) { + post_phi_calculation.value()(mlmg, lev); + } } } + rho[lev]->mult(-ablastr::constant::SI::ep0); // Multiply rho by epsilon again } // loop over lev(els) From 98584dfae209a9d3d729faebc35d1d3ed2f43465 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Mon, 16 Dec 2024 17:11:39 -0800 Subject: [PATCH 31/37] gmres fluxes for E calc --- Source/ablastr/fields/PoissonSolver.H | 55 +++++++++++++-------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index e8ba35743cc..f3caf448f32 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -489,34 +489,6 @@ computePhi ( gmsolve.solve( *phi[lev], *rho[lev], relative_tolerance, absolute_tolerance ); } - auto & warpx = WarpX::GetInstance(); - // EB: use AMReX to directly calculate the electric field since with EB's the - // simple finite difference scheme in WarpX::computeE sometimes fails - - // TODO: maybe make this a helper function or pass Efield_fp directly - amrex::Vector< amrex::Array > e_field; - e_field.push_back( -#if defined(WARPX_DIM_1D_Z) - amrex::Array{ - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) - } -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Array{ - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, 0), - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) - } -#elif defined(WARPX_DIM_3D) - amrex::Array{ - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, 0), - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{1}, 0), - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) - } -#endif - ); - - - mlmg.getFluxes(e_field, phi); - const auto mlmg_time_end_step = static_cast(amrex::second()); amrex::Print()<< "MLMG time = " @@ -548,6 +520,33 @@ computePhi ( post_phi_calculation.value()(mlmg, lev); } } + } else { + + auto & warpx = WarpX::GetInstance(); + // EB: use AMReX to directly calculate the electric field since with EB's the + // simple finite difference scheme in WarpX::computeE sometimes fails + + // TODO: maybe make this a helper function or pass Efield_fp directly + amrex::Vector< amrex::Array > e_field; + e_field.push_back( +#if defined(WARPX_DIM_1D_Z) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) + } +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, 0), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) + } +#elif defined(WARPX_DIM_3D) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, 0), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{1}, 0), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) + } +#endif + ); + linop->compGrad(0, e_field[0], *phi[0], amrex::MLLinOp::Location::CellCenter); } rho[lev]->mult(-ablastr::constant::SI::ep0); // Multiply rho by epsilon again From cc9d72463927e3c5809cdcf67d56f54e908a0ee0 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Tue, 17 Dec 2024 10:14:10 -0800 Subject: [PATCH 32/37] multiply grad(phi) = E fluxes by -1 for the gmres case --- Source/ablastr/fields/PoissonSolver.H | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index f3caf448f32..dd7d24cf905 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -547,6 +547,9 @@ computePhi ( #endif ); linop->compGrad(0, e_field[0], *phi[0], amrex::MLLinOp::Location::CellCenter); + for(int dir=0; dirmult(-1.); + } } rho[lev]->mult(-ablastr::constant::SI::ep0); // Multiply rho by epsilon again From e8a328bafeb57fd0a0d4a926e1d4e2005953ce98 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Tue, 17 Dec 2024 12:55:54 -0800 Subject: [PATCH 33/37] debugging --- Source/ablastr/fields/PoissonSolver.H | 59 +++++++++++++++------------ 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index dd7d24cf905..4b07d8be92f 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -259,7 +259,6 @@ computePhi ( amrex::LPInfo info; - amrex::Print() << "HERE LPInfo\n"; amrex::ParmParse pp1("lpinfo"); int aaa; @@ -424,8 +423,6 @@ computePhi ( mlmg.setMaxIter(max_iters); mlmg.setAlwaysUseBNorm((max_norm_b > 0)); - amrex::Print() << "HERE MLMG\n"; - amrex::ParmParse pp2("mlmg"); int xxx; amrex::Real yyy; @@ -480,15 +477,19 @@ computePhi ( const auto mlmg_time_beg_step = static_cast(amrex::second()); - amrex::GMRESMLMG gmsolve(mlmg); - // Solve Poisson equation at lev if (solver_type == 0) { mlmg.solve( {phi[lev]}, {rho[lev]}, relative_tolerance, absolute_tolerance ); } else { + amrex::GMRESMLMG gmsolve(mlmg); gmsolve.solve( *phi[lev], *rho[lev], relative_tolerance, absolute_tolerance ); } +#if 0 + amrex::VisMF::Write(*rho[lev],"rho"); + amrex::VisMF::Write(*phi[lev],"phi"); +#endif + const auto mlmg_time_end_step = static_cast(amrex::second()); amrex::Print()<< "MLMG time = " @@ -520,38 +521,46 @@ computePhi ( post_phi_calculation.value()(mlmg, lev); } } - } else { + } - auto & warpx = WarpX::GetInstance(); - // EB: use AMReX to directly calculate the electric field since with EB's the - // simple finite difference scheme in WarpX::computeE sometimes fails + // create an alieas to the WarpX Efield + auto & warpx = WarpX::GetInstance(); - // TODO: maybe make this a helper function or pass Efield_fp directly - amrex::Vector< amrex::Array > e_field; - e_field.push_back( + amrex::Vector< amrex::Array > e_field; + e_field.push_back( #if defined(WARPX_DIM_1D_Z) - amrex::Array{ - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, lev) } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Array{ - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, 0), - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) - } + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, lev) + } #elif defined(WARPX_DIM_3D) - amrex::Array{ - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, 0), - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{1}, 0), - warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, 0) - } + amrex::Array{ + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{1}, lev), + warpx.m_fields.get(warpx::fields::FieldType::Efield_fp, Direction{2}, lev) + } #endif - ); + ); + + if (solver_type == 1) { linop->compGrad(0, e_field[0], *phi[0], amrex::MLLinOp::Location::CellCenter); for(int dir=0; dirmult(-1.); + e_field[lev][dir]->mult(-1.); } } +#if 0 + amrex::VisMF::Write(*e_field[lev][0],"Ex"); + amrex::VisMF::Write(*e_field[lev][1],"Ey"); +#if defined(WARPX_DIM_3D) + amrex::VisMF::Write(*e_field[lev][2],"Ez"); +#endif +#endif + rho[lev]->mult(-ablastr::constant::SI::ep0); // Multiply rho by epsilon again } // loop over lev(els) From 5a3eee6298735bfae39b9dd2c4525fe9ac6eb534 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Fri, 20 Dec 2024 10:44:27 -0800 Subject: [PATCH 34/37] postSolve to zero out EB phi --- Source/ablastr/fields/PoissonSolver.H | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index ec7e30791e6..82343c6d9bf 100755 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -486,6 +486,8 @@ computePhi ( gmsolve.solve( *phi[lev], *rho[lev], relative_tolerance, absolute_tolerance ); } + linop->postSolve({phi[lev]}); + #if 0 amrex::VisMF::Write(*rho[lev],"rho"); amrex::VisMF::Write(*phi[lev],"phi"); From da42dc14fecd790812349839d2d52b8b3cb52f02 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Fri, 20 Dec 2024 11:24:08 -0800 Subject: [PATCH 35/37] 256 cpu cases --- run_plasma/inputs.2d_256cpu | 130 ++++++++++++++++++++++++++++++++++++ run_plasma/run_256cpu.sh | 15 +++++ 2 files changed, 145 insertions(+) create mode 100644 run_plasma/inputs.2d_256cpu create mode 100644 run_plasma/run_256cpu.sh diff --git a/run_plasma/inputs.2d_256cpu b/run_plasma/inputs.2d_256cpu new file mode 100644 index 00000000000..c32b7f9b394 --- /dev/null +++ b/run_plasma/inputs.2d_256cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 2048 1024 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/run_256cpu.sh b/run_plasma/run_256cpu.sh new file mode 100644 index 00000000000..68d52310a2c --- /dev/null +++ b/run_plasma/run_256cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 2 +#SBATCH -C cpu +#SBATCH -q debug +#SBATCH -J 256cpu +#SBATCH -t 00:30:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 256 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_256cpu + From f204eb24b0a73837c38cbebfdc8e40881945eb52 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Wed, 8 Jan 2025 09:31:09 -0800 Subject: [PATCH 36/37] linop changes for compilation --- Source/ablastr/fields/PoissonSolver.H | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 82343c6d9bf..ca9641782f9 100755 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -484,10 +484,11 @@ computePhi ( } else { amrex::GMRESMLMG gmsolve(mlmg); gmsolve.solve( *phi[lev], *rho[lev], relative_tolerance, absolute_tolerance ); + amrex::Vector vmf; + vmf.emplace_back(*phi[lev], amrex::make_alias, 0, phi[lev]->nComp()); + linop->postSolve(vmf); } - linop->postSolve({phi[lev]}); - #if 0 amrex::VisMF::Write(*rho[lev],"rho"); amrex::VisMF::Write(*phi[lev],"phi"); From c7110b8a1bf5d36e55b83979aa99b2acd8493084 Mon Sep 17 00:00:00 2001 From: Andy Nonaka Date: Fri, 31 Jan 2025 16:16:19 -0800 Subject: [PATCH 37/37] full scaling tests --- run_plasma/CPU/inputs.2d_1024cpu | 130 ++++++++++++++++++++++++++ run_plasma/CPU/inputs.2d_16384cpu | 130 ++++++++++++++++++++++++++ run_plasma/{ => CPU}/inputs.2d_256cpu | 0 run_plasma/CPU/inputs.2d_4096cpu | 130 ++++++++++++++++++++++++++ run_plasma/{ => CPU}/inputs.2d_64cpu | 0 run_plasma/CPU/inputs.2d_65536cpu | 130 ++++++++++++++++++++++++++ run_plasma/CPU/run_1024cpu.sh | 15 +++ run_plasma/CPU/run_16384cpu.sh | 15 +++ run_plasma/{ => CPU}/run_256cpu.sh | 2 +- run_plasma/CPU/run_4096cpu.sh | 15 +++ run_plasma/{ => CPU}/run_64cpu.sh | 0 run_plasma/CPU/run_65536cpu.sh | 15 +++ run_plasma/{ => GPU}/inputs.2d_128gpu | 0 run_plasma/GPU/inputs.2d_2048gpu | 130 ++++++++++++++++++++++++++ run_plasma/{ => GPU}/inputs.2d_2gpu | 0 run_plasma/{ => GPU}/inputs.2d_32gpu | 0 run_plasma/GPU/inputs.2d_512gpu | 130 ++++++++++++++++++++++++++ run_plasma/{ => GPU}/inputs.2d_8gpu | 0 run_plasma/{ => GPU}/run_128gpu.sh | 0 run_plasma/GPU/run_2048gpu.sh | 19 ++++ run_plasma/{ => GPU}/run_2gpu.sh | 0 run_plasma/{ => GPU}/run_32gpu.sh | 3 +- run_plasma/GPU/run_512gpu.sh | 19 ++++ run_plasma/{ => GPU}/run_8gpu.sh | 3 +- 24 files changed, 881 insertions(+), 5 deletions(-) create mode 100644 run_plasma/CPU/inputs.2d_1024cpu create mode 100644 run_plasma/CPU/inputs.2d_16384cpu rename run_plasma/{ => CPU}/inputs.2d_256cpu (100%) create mode 100644 run_plasma/CPU/inputs.2d_4096cpu rename run_plasma/{ => CPU}/inputs.2d_64cpu (100%) create mode 100644 run_plasma/CPU/inputs.2d_65536cpu create mode 100644 run_plasma/CPU/run_1024cpu.sh create mode 100644 run_plasma/CPU/run_16384cpu.sh rename run_plasma/{ => CPU}/run_256cpu.sh (93%) create mode 100644 run_plasma/CPU/run_4096cpu.sh rename run_plasma/{ => CPU}/run_64cpu.sh (100%) create mode 100644 run_plasma/CPU/run_65536cpu.sh rename run_plasma/{ => GPU}/inputs.2d_128gpu (100%) create mode 100644 run_plasma/GPU/inputs.2d_2048gpu rename run_plasma/{ => GPU}/inputs.2d_2gpu (100%) rename run_plasma/{ => GPU}/inputs.2d_32gpu (100%) create mode 100644 run_plasma/GPU/inputs.2d_512gpu rename run_plasma/{ => GPU}/inputs.2d_8gpu (100%) rename run_plasma/{ => GPU}/run_128gpu.sh (100%) create mode 100644 run_plasma/GPU/run_2048gpu.sh rename run_plasma/{ => GPU}/run_2gpu.sh (100%) rename run_plasma/{ => GPU}/run_32gpu.sh (72%) create mode 100644 run_plasma/GPU/run_512gpu.sh rename run_plasma/{ => GPU}/run_8gpu.sh (72%) diff --git a/run_plasma/CPU/inputs.2d_1024cpu b/run_plasma/CPU/inputs.2d_1024cpu new file mode 100644 index 00000000000..880b930a6ad --- /dev/null +++ b/run_plasma/CPU/inputs.2d_1024cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 4096 2048 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/inputs.2d_16384cpu b/run_plasma/CPU/inputs.2d_16384cpu new file mode 100644 index 00000000000..a03622ff40a --- /dev/null +++ b/run_plasma/CPU/inputs.2d_16384cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 16384 8192 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/inputs.2d_256cpu b/run_plasma/CPU/inputs.2d_256cpu similarity index 100% rename from run_plasma/inputs.2d_256cpu rename to run_plasma/CPU/inputs.2d_256cpu diff --git a/run_plasma/CPU/inputs.2d_4096cpu b/run_plasma/CPU/inputs.2d_4096cpu new file mode 100644 index 00000000000..0efbfef24bc --- /dev/null +++ b/run_plasma/CPU/inputs.2d_4096cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 8192 4096 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/inputs.2d_64cpu b/run_plasma/CPU/inputs.2d_64cpu similarity index 100% rename from run_plasma/inputs.2d_64cpu rename to run_plasma/CPU/inputs.2d_64cpu diff --git a/run_plasma/CPU/inputs.2d_65536cpu b/run_plasma/CPU/inputs.2d_65536cpu new file mode 100644 index 00000000000..488f7efd962 --- /dev/null +++ b/run_plasma/CPU/inputs.2d_65536cpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 32768 16384 +amr.max_grid_size_x = 128 +amr.max_grid_size_y = 64 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/CPU/run_1024cpu.sh b/run_plasma/CPU/run_1024cpu.sh new file mode 100644 index 00000000000..bf60a12dc45 --- /dev/null +++ b/run_plasma/CPU/run_1024cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 8 +#SBATCH -C cpu +#SBATCH -q debug +#SBATCH -J 1024cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 1024 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_1024cpu + diff --git a/run_plasma/CPU/run_16384cpu.sh b/run_plasma/CPU/run_16384cpu.sh new file mode 100644 index 00000000000..7bdb031c37b --- /dev/null +++ b/run_plasma/CPU/run_16384cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 128 +#SBATCH -C cpu +#SBATCH -q regular +#SBATCH -J 16384cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 16384 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_16384cpu + diff --git a/run_plasma/run_256cpu.sh b/run_plasma/CPU/run_256cpu.sh similarity index 93% rename from run_plasma/run_256cpu.sh rename to run_plasma/CPU/run_256cpu.sh index 68d52310a2c..0089374ee25 100644 --- a/run_plasma/run_256cpu.sh +++ b/run_plasma/CPU/run_256cpu.sh @@ -3,7 +3,7 @@ #SBATCH -C cpu #SBATCH -q debug #SBATCH -J 256cpu -#SBATCH -t 00:30:00 +#SBATCH -t 00:05:00 #OpenMP settings: export OMP_NUM_THREADS=1 diff --git a/run_plasma/CPU/run_4096cpu.sh b/run_plasma/CPU/run_4096cpu.sh new file mode 100644 index 00000000000..08850f2013c --- /dev/null +++ b/run_plasma/CPU/run_4096cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 32 +#SBATCH -C cpu +#SBATCH -q regular +#SBATCH -J 4096cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 4096 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_4096cpu + diff --git a/run_plasma/run_64cpu.sh b/run_plasma/CPU/run_64cpu.sh similarity index 100% rename from run_plasma/run_64cpu.sh rename to run_plasma/CPU/run_64cpu.sh diff --git a/run_plasma/CPU/run_65536cpu.sh b/run_plasma/CPU/run_65536cpu.sh new file mode 100644 index 00000000000..8801c926fd1 --- /dev/null +++ b/run_plasma/CPU/run_65536cpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -N 512 +#SBATCH -C cpu +#SBATCH -q regular +#SBATCH -J 65536cpu +#SBATCH -t 00:05:00 + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run the application: +srun -n 65536 -c 2 --cpu_bind=cores ./warpx.2d.MPI.OMP.DP.PDP.OPMD.EB inputs.2d_65536cpu + diff --git a/run_plasma/inputs.2d_128gpu b/run_plasma/GPU/inputs.2d_128gpu similarity index 100% rename from run_plasma/inputs.2d_128gpu rename to run_plasma/GPU/inputs.2d_128gpu diff --git a/run_plasma/GPU/inputs.2d_2048gpu b/run_plasma/GPU/inputs.2d_2048gpu new file mode 100644 index 00000000000..e35de2ff27a --- /dev/null +++ b/run_plasma/GPU/inputs.2d_2048gpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 32768 16384 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/inputs.2d_2gpu b/run_plasma/GPU/inputs.2d_2gpu similarity index 100% rename from run_plasma/inputs.2d_2gpu rename to run_plasma/GPU/inputs.2d_2gpu diff --git a/run_plasma/inputs.2d_32gpu b/run_plasma/GPU/inputs.2d_32gpu similarity index 100% rename from run_plasma/inputs.2d_32gpu rename to run_plasma/GPU/inputs.2d_32gpu diff --git a/run_plasma/GPU/inputs.2d_512gpu b/run_plasma/GPU/inputs.2d_512gpu new file mode 100644 index 00000000000..bd882ece0ac --- /dev/null +++ b/run_plasma/GPU/inputs.2d_512gpu @@ -0,0 +1,130 @@ +amrex.use_gpu_aware_mpi = 1 + +mlmg.setPreSmooth = 2 +mlmg.setPostSmooth = 2 + +mlmg.setFinalSmooth = 8 +mlmg.setBottomSmooth = 0 + +mlmg.bottomSolver = 0 +mlmg.setBottomTolerance = 1.e-4 + +lpinfo.setMaxCoarseningLevel = 30 + +# Argon +# We want to simulate Figs. 4 (b), 5 (a) and (b), and 7 (a) in Rauf et al. 2020. + +my_constants.Ngas = 3.22e20 # 100 mTorr # (m^-3) +my_constants.Tgas = 300. # (K) +my_constants.Te = 23212. # (K) see Chen et al. 2024 +my_constants.Nplasma = 5.e16 # (m^-3) see Chen et al. 2024 +my_constants.freq = 13.56e6 # (Hz) +my_constants.Mion = 6.63e-26 # (kg) +my_constants.voltage = 100. # (V) +my_constants.clight = 3.e8 # speed of light in vacuum +my_constants.m_e = 9.11e-31 # (kg) +my_constants.kb = 1.38e-23 # (J/K) + +# amr.restart = ./diags/chk01450000 +max_step = 100 # 2000000 # 5000 RF cycles +warpx.verbose = 1 +warpx.const_dt = 1.0/(400*freq) +warpx.do_electrostatic = labframe +warpx.self_fields_required_precision = 1.e-7 # +warpx.use_filter = 0 +warpx.sort_intervals = -1 + +amr.n_cell = 16384 8192 +amr.max_grid_size_x = 512 +amr.max_grid_size_y = 512 +amr.blocking_factor = 8 +amr.max_level = 0 + +geometry.dims = 2 +geometry.prob_lo = -0.1035 -0.0527 # x z # cover complete chamber (do not exploit symmetry) +geometry.prob_hi = 0.1035 0.0527 +boundary.field_lo = pec pec +boundary.field_hi = pec pec +boundary.potential_hi_x = 0. +boundary.potential_lo_z = 0. +boundary.potential_lo_x = 0. +boundary.potential_hi_z = 0. +boundary.particle_lo = reflecting reflecting +boundary.particle_hi = reflecting reflecting + +# Order of particle shape factors +algo.particle_shape = 1 + +# EB +my_constants.te_xmax = 0.0488 +my_constants.dx_thick = 0.0032 # dielectric thickness +my_constants.be_xmax = 0.052 +my_constants.zhi = 0.0128 +my_constants.zlo = -0.0128 +warpx.eb_implicit_function = "min(max((zlo-z),(z-zhi)),-max((x+(-be_xmax)),-(x+be_xmax)))" # "if( ((z>zhi) or (z-be_xmax) , 1,-1 )" # ?? +warpx.eb_potential(x,y,z,t) = " sin(2*pi*freq*t)*(voltage*(z>zhi)*(x-te_xmax) + voltage*(z>zhi)*(x>te_xmax)*(xzhi)*(x<-te_xmax)*(x>-be_xmax)*(x+be_xmax)/dx_thick + 0.*(z-be_xmax)) " + +particles.species_names = electrons ar_ions + +electrons.species_type = electron +electrons.injection_style = nuniformpercell +electrons.initialize_self_fields = 0 +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = Nplasma +electrons.momentum_distribution_type = maxwell_boltzmann +electrons.theta = (kb*Te/(m_e*clight^2)) + +ar_ions.species_type = argon +ar_ions.charge = q_e +ar_ions.injection_style = nuniformpercell +ar_ions.initialize_self_fields = 0 +ar_ions.num_particles_per_cell_each_dim = 8 8 +ar_ions.profile = constant +ar_ions.density = Nplasma +ar_ions.momentum_distribution_type = maxwell_boltzmann +ar_ions.theta = (kb*Tgas/(Mion*clight^2)) +ar_ions.save_particles_at_eb = 1 + +#collisions.collision_names = coll_elec coll_ion +coll_ion.type = background_mcc +coll_ion.species = ar_ions +coll_ion.background_density = Ngas +coll_ion.background_temperature = Tgas +coll_ion.scattering_processes = elastic back charge_exchange +coll_ion.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_scattering.dat +coll_ion.back_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ion_back_scatter.dat +coll_ion.charge_exchange_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/charge_exchange.dat + +coll_elec.type = background_mcc +coll_elec.species = electrons +coll_elec.background_density = Ngas +coll_elec.background_temperature = Tgas +coll_elec.scattering_processes = elastic excitation1 ionization +coll_elec.elastic_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/electron_scattering.dat +coll_elec.excitation1_energy = 11.5 +coll_elec.excitation1_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/excitation_1.dat +coll_elec.ionization_energy = 15.7596112 +coll_elec.ionization_cross_section = /global/cfs/cdirs/mp111/warpx-data/MCC_cross_sections/Ar/ionization.dat +coll_elec.ionization_species = ar_ions + +#diagnostics.diags_names = plt chk #plt_eb +#plt.diag_type = Full +#plt.intervals = 1 +#plt.fields_to_plot = phi +#plt.file_min_digits = 8 + +#plt_eb.diag_type = BoundaryScraping +#plt_eb.format = openpmd +#plt_eb.fields_to_plot = phi +#plt_eb.particle_field_species = ar_ions +#plt_eb.intervals = 190000:200000:1000 + +#chk.diag_type = Full +#chk.format = checkpoint +#chk.intervals = 1000 +#chk.file_min_digits = 8 + +#warpx.reduced_diags_names = partnum +#partnum.type = ParticleNumber +#partnum.intervals = 1 diff --git a/run_plasma/inputs.2d_8gpu b/run_plasma/GPU/inputs.2d_8gpu similarity index 100% rename from run_plasma/inputs.2d_8gpu rename to run_plasma/GPU/inputs.2d_8gpu diff --git a/run_plasma/run_128gpu.sh b/run_plasma/GPU/run_128gpu.sh similarity index 100% rename from run_plasma/run_128gpu.sh rename to run_plasma/GPU/run_128gpu.sh diff --git a/run_plasma/GPU/run_2048gpu.sh b/run_plasma/GPU/run_2048gpu.sh new file mode 100644 index 00000000000..1d0091ee52f --- /dev/null +++ b/run_plasma/GPU/run_2048gpu.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH -N 512 +#SBATCH -C gpu +#SBATCH -G 2048 +#SBATCH -q regular +#SBATCH -J 2048gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 2048 -c 32 --cpu_bind=cores -G 2048 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_2048gpu + diff --git a/run_plasma/run_2gpu.sh b/run_plasma/GPU/run_2gpu.sh similarity index 100% rename from run_plasma/run_2gpu.sh rename to run_plasma/GPU/run_2gpu.sh diff --git a/run_plasma/run_32gpu.sh b/run_plasma/GPU/run_32gpu.sh similarity index 72% rename from run_plasma/run_32gpu.sh rename to run_plasma/GPU/run_32gpu.sh index 466cc8bbcf3..1afffb1177c 100644 --- a/run_plasma/run_32gpu.sh +++ b/run_plasma/GPU/run_32gpu.sh @@ -15,5 +15,4 @@ export OMP_PROC_BIND=spread # pin to closest NIC to GPU export MPICH_OFI_NIC_POLICY=GPU -srun -n 32 -c 32 --cpu_bind=cores -G 32 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_32gpu - +srun -n 32 -c 32 --cpu_bind=cores -G 32 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_32gpu diff --git a/run_plasma/GPU/run_512gpu.sh b/run_plasma/GPU/run_512gpu.sh new file mode 100644 index 00000000000..39d2b70cd89 --- /dev/null +++ b/run_plasma/GPU/run_512gpu.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH -N 128 +#SBATCH -C gpu +#SBATCH -G 512 +#SBATCH -q regular +#SBATCH -J 512gpu +#SBATCH -t 00:05:00 +#SBATCH -A mp111_g + +#OpenMP settings: +export OMP_NUM_THREADS=1 +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +# pin to closest NIC to GPU +export MPICH_OFI_NIC_POLICY=GPU + +srun -n 512 -c 32 --cpu_bind=cores -G 512 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_512gpu + diff --git a/run_plasma/run_8gpu.sh b/run_plasma/GPU/run_8gpu.sh similarity index 72% rename from run_plasma/run_8gpu.sh rename to run_plasma/GPU/run_8gpu.sh index fa3eed7d8a7..369ceb317bb 100644 --- a/run_plasma/run_8gpu.sh +++ b/run_plasma/GPU/run_8gpu.sh @@ -15,5 +15,4 @@ export OMP_PROC_BIND=spread # pin to closest NIC to GPU export MPICH_OFI_NIC_POLICY=GPU -srun -n 8 -c 32 --cpu_bind=cores -G 8 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_8gpu - +srun -n 8 -c 32 --cpu_bind=cores -G 8 --gpu-bind=none ./warpx.2d.MPI.CUDA.DP.PDP.OPMD.EB inputs.2d_8gpu