From 865ba6372681500116e2484465d0206a13180c88 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 15 May 2024 10:37:07 +0200 Subject: [PATCH] fix race conditions --- Source/BoundaryConditions/WarpX_PEC.cpp | 6 +++--- .../Particles/PhysicalParticleContainer.cpp | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index a42873db555..0067f54d3ff 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -840,6 +840,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Each current component is handled separately below, starting with Jx. + grown_domain_box.convert(Jx_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -850,7 +851,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jx_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -875,6 +875,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Handle Jy. + grown_domain_box.convert(Jy_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -885,7 +886,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jy_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -910,6 +910,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Handle Jz. + grown_domain_box.convert(Jz_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -920,7 +921,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jz_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c7f44903d9b..3d134393f06 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1003,6 +1003,19 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const bool radially_weighted = plasma_injector.radially_weighted; #endif + + // User-defined integer and real attributes: prepare parsers + const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); + const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); + for (int ia = 0; ia < n_user_int_attribs; ++ia) { + user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); + } + for (int ia = 0; ia < n_user_real_attribs; ++ia) { + user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); + } + MFItInfo info; if (do_tiling && Gpu::notInLaunchRegion()) { info.EnableTiling(tile_size); @@ -1156,19 +1169,13 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int } uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); amrex::Gpu::PinnedVector pa_user_int_pinned(n_user_int_attribs); amrex::Gpu::PinnedVector pa_user_real_pinned(n_user_real_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); for (int ia = 0; ia < n_user_int_attribs; ++ia) { pa_user_int_pinned[ia] = soa.GetIntData(particle_icomps[m_user_int_attribs[ia]]).data() + old_size; - user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); } for (int ia = 0; ia < n_user_real_attribs; ++ia) { pa_user_real_pinned[ia] = soa.GetRealData(particle_comps[m_user_real_attribs[ia]]).data() + old_size; - user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); } #ifdef AMREX_USE_GPU // To avoid using managed memory, we first define pinned memory vector, initialize on cpu,