diff --git a/pyphare/pyphare/pharein/__init__.py b/pyphare/pyphare/pharein/__init__.py index 13934701c..bcc78a7d7 100644 --- a/pyphare/pyphare/pharein/__init__.py +++ b/pyphare/pyphare/pharein/__init__.py @@ -182,6 +182,8 @@ def add_vector_int(path, val): add_int("simulation/AMR/tag_buffer", simulation.tag_buffer) + add_string("simulation/AMR/loadbalancing", simulation.loadbalancing) + refinement_boxes = simulation.refinement_boxes def as_paths(rb): diff --git a/pyphare/pyphare/pharein/simulation.py b/pyphare/pyphare/pharein/simulation.py index 1855004d5..448376489 100644 --- a/pyphare/pyphare/pharein/simulation.py +++ b/pyphare/pyphare/pharein/simulation.py @@ -572,6 +572,16 @@ def check_clustering(**kwargs): return clustering +def check_loadbalancing(**kwargs): + valid_keys = ["nppc", "homogeneous"] + loadbalancing = kwargs.get("loadbalancing", "nppc") + if loadbalancing not in valid_keys: + raise ValueError( + f"Error: loadbalancing type is not supported, supported types are {valid_keys}" + ) + return loadbalancing + + # ------------------------------------------------------------------------------ @@ -605,6 +615,7 @@ def wrapper(simulation_object, **kwargs): "restart_options", "tag_buffer", "description", + "loadbalancing", ] accepted_keywords += check_optional_keywords(**kwargs) @@ -625,6 +636,8 @@ def wrapper(simulation_object, **kwargs): kwargs["clustering"] = check_clustering(**kwargs) + kwargs["loadbalancing"] = check_loadbalancing(**kwargs) + time_step_nbr, time_step, final_time = check_time(**kwargs) kwargs["time_step_nbr"] = time_step_nbr kwargs["time_step"] = time_step diff --git a/src/amr/CMakeLists.txt b/src/amr/CMakeLists.txt index 82c52e220..83eed7bbd 100644 --- a/src/amr/CMakeLists.txt +++ b/src/amr/CMakeLists.txt @@ -65,6 +65,13 @@ set( SOURCES_INC level_initializer/hybrid_level_initializer.hpp level_initializer/level_initializer_factory.hpp data/field/field_variable_fill_pattern.hpp + load_balancing/load_balancer_manager.hpp + load_balancing/load_balancer_estimator.hpp + load_balancing/load_balancer_estimator_hybrid.hpp + load_balancing/load_balancer_hybrid_strategy_factory.hpp + load_balancing/load_balancer_hybrid_strategy.hpp + load_balancing/concrete_load_balancer_hybrid_strategy_homogeneous.hpp + load_balancing/concrete_load_balancer_hybrid_strategy_nppc.hpp ) set( SOURCES_CPP data/field/refine/linear_weighter.cpp diff --git a/src/amr/load_balancing/concrete_load_balancer_hybrid_strategy_homogeneous.hpp b/src/amr/load_balancing/concrete_load_balancer_hybrid_strategy_homogeneous.hpp new file mode 100644 index 000000000..ab93c7ea2 --- /dev/null +++ b/src/amr/load_balancing/concrete_load_balancer_hybrid_strategy_homogeneous.hpp @@ -0,0 +1,139 @@ + + +#ifndef PHARE_CONCRERTE_LOAD_BALANCER_HYBRID_STRATEGY_HOMOGENEOUS_HPP +#define PHARE_CONCRERTE_LOAD_BALANCER_HYBRID_STRATEGY_HOMOGENEOUS_HPP + +#include +#include +#include + +#include "amr/load_balancing/load_balancer_hybrid_strategy.hpp" +#include "amr/physical_models/physical_model.hpp" +#include "amr/types/amr_types.hpp" +#include "amr/resources_manager/amr_utils.hpp" +#include "core/data/ndarray/ndarray_vector.hpp" + + + + +namespace PHARE::amr +{ +template +class ConcreteLoadBalancerHybridStrategyHomogeneous : public LoadBalancerHybridStrategy +{ +public: + using HybridModel = typename PHARE_T::HybridModel_t; + using gridlayout_type = typename HybridModel::gridlayout_type; + + ConcreteLoadBalancerHybridStrategyHomogeneous(int const id) + : id_{id} + { + } + + void compute(SAMRAI::hier::PatchLevel& level, + PHARE::solver::IPhysicalModel& model) override; + + +private: + int const id_; +}; + + + +template +void ConcreteLoadBalancerHybridStrategyHomogeneous::compute( + SAMRAI::hier::PatchLevel& level, PHARE::solver::IPhysicalModel& model) +{ + static auto constexpr dimension = HybridModel::dimension; + auto& hybridModel = dynamic_cast(model); + auto& resourcesManager = hybridModel.resourcesManager; + auto& ions = hybridModel.state.ions; + + bool constexpr c_ordering = false; + + for (auto& patch : level) + { + auto const& layout = layoutFromPatch(*patch); + + auto patch_data_lb + = dynamic_cast*>(patch->getPatchData(this->id_).get()); + auto load_balancer_val = patch_data_lb->getPointer(); + + const auto& box = patch->getBox(); + + auto lb_view = core::NdArrayView(load_balancer_val, + layout.nbrCells()); + + auto _ = resourcesManager->setOnPatch(*patch, ions); + + // The view on "load_balancer_val" do not have any ghost cells, meaning that this patch + // data lies on a box defind by the nbr of cells in the physical domain + // As the nbr of ghost cells in the box associated to the load balancer patch data is + // null, we hence loop on an AMR index (then needing to firstly get the AMR box) + // to get the AMRLocatStart and AMRLocalEnd. + // Then, we build "by hand" the local index for the "lb_view" considering that the nbr + // of ghost cell for this patch data is null. + auto amr_box = layout.AMRBox(); + + // The lb_view is a CellData, meaning that it is dual as the index of an amr box + auto amr_start = amr_box.lower; + auto amr_end = amr_box.upper; + + // nbr of cells in the physical domain + auto nbrCells = layout.nbrCells(); + + + + + if constexpr (dimension == 1) + { + for (auto i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + { + auto amr_point{core::Point{i_amr}}; + auto amr_index = amr_point.template toArray(); + + lb_view(i_loc) = 1; + } + } + + + + if constexpr (dimension == 2) + { + for (auto i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + { + for (auto j_loc = 0, j_amr = amr_start[1]; j_loc < nbrCells[1]; ++j_loc, ++j_amr) + { + auto amr_point{core::Point{i_amr, j_amr}}; + auto amr_index = amr_point.template toArray(); + + lb_view(i_loc, j_loc) = 1; + std::cout << lb_view(i_loc, j_loc) << std::endl; + } + } + } + + + if constexpr (dimension == 3) + { + for (auto i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + { + for (auto j_loc = 0, j_amr = amr_start[1]; j_loc < nbrCells[1]; ++j_loc, ++j_amr) + { + for (auto k_loc = 0, k_amr = amr_start[2]; k_loc < nbrCells[2]; + ++k_loc, ++k_amr) + { + auto amr_point{core::Point{i_amr, j_amr, k_amr}}; + auto amr_index = amr_point.template toArray(); + + lb_view(i_loc, j_loc, k_loc) = 1; + } + } + } + } + } +} + +} // namespace PHARE::amr + +#endif diff --git a/src/amr/load_balancing/concrete_load_balancer_hybrid_strategy_nppc.hpp b/src/amr/load_balancing/concrete_load_balancer_hybrid_strategy_nppc.hpp new file mode 100644 index 000000000..6965a0f1a --- /dev/null +++ b/src/amr/load_balancing/concrete_load_balancer_hybrid_strategy_nppc.hpp @@ -0,0 +1,173 @@ + + +#ifndef PHARE_CONCRERTE_LOAD_BALANCER_HYBRID_STRATEGY_NPPC_HPP +#define PHARE_CONCRERTE_LOAD_BALANCER_HYBRID_STRATEGY_NPPC_HPP + +#include +#include + +#include + +#include "amr/load_balancing/load_balancer_hybrid_strategy.hpp" +#include "amr/physical_models/physical_model.hpp" +#include "amr/types/amr_types.hpp" +#include "amr/resources_manager/amr_utils.hpp" +#include "core/data/ndarray/ndarray_vector.hpp" + + + + +namespace PHARE::amr +{ +template +class ConcreteLoadBalancerHybridStrategyNPPC : public LoadBalancerHybridStrategy +{ +public: + using HybridModel = typename PHARE_T::HybridModel_t; + using gridlayout_type = typename HybridModel::gridlayout_type; + + ConcreteLoadBalancerHybridStrategyNPPC(int const id) + : id_{id} + { + } + + void compute(SAMRAI::hier::PatchLevel& level, + PHARE::solver::IPhysicalModel& model) override; + + +private: + int const id_; +}; + + + +template +void ConcreteLoadBalancerHybridStrategyNPPC::compute( + SAMRAI::hier::PatchLevel& level, PHARE::solver::IPhysicalModel& model) +{ + static auto constexpr dimension = HybridModel::dimension; + auto& hybridModel = dynamic_cast(model); + auto& resourcesManager = hybridModel.resourcesManager; + auto& ions = hybridModel.state.ions; + + bool constexpr c_ordering = false; + + for (auto& patch : level) + { + auto const& layout = layoutFromPatch(*patch); + + auto patch_data_lb + = dynamic_cast*>(patch->getPatchData(this->id_).get()); + auto load_balancer_val = patch_data_lb->getPointer(); + + const auto& box = patch->getBox(); + + auto lb_view = core::NdArrayView(load_balancer_val, + layout.nbrCells()); + + auto _ = resourcesManager->setOnPatch(*patch, ions); + + // The view on "load_balancer_val" do not have any ghost cells, meaning that this patch + // data lies on a box defind by the nbr of cells in the physical domain + // As the nbr of ghost cells in the box associated to the load balancer patch data is + // null, we hence loop on an AMR index (then needing to firstly get the AMR box) + // to get the AMRLocatStart and AMRLocalEnd. + // Then, we build "by hand" the local index for the "lb_view" considering that the nbr + // of ghost cell for this patch data is null. + auto amr_box = layout.AMRBox(); + + // The lb_view is a CellData, meaning that it is dual as the index of an amr box + auto amr_start = amr_box.lower; + auto amr_end = amr_box.upper; + + // nbr of cells in the physical domain + auto nbrCells = layout.nbrCells(); + + + + + if constexpr (dimension == 1) + { + for (std::uint32_t i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + { + auto amr_point{core::Point{i_amr}}; + auto amr_index = amr_point.template toArray(); + + std::size_t nbr = 0; + + for (auto& pop : ions) + { + const auto& domainParticles = pop.domainParticles(); + + nbr += domainParticles.nbr_particles_in(amr_index); + } + + lb_view(i_loc) = nbr; + } + } + + + + if constexpr (dimension == 2) + { + for (std::uint32_t i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + { + for (std::uint32_t j_loc = 0, j_amr = amr_start[1]; j_loc < nbrCells[1]; ++j_loc, ++j_amr) + { + auto amr_point{core::Point{i_amr, j_amr}}; + auto amr_index = amr_point.template toArray(); + + std::size_t nbr = 0; + + for (auto& pop : ions) + { + const auto& domainParticles = pop.domainParticles(); + + nbr += domainParticles.nbr_particles_in(amr_index); + } + + lb_view(i_loc, j_loc) = nbr; + } + } + } + + + if constexpr (dimension == 3) + { + for (std::uint32_t i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + { + for (std::uint32_t j_loc = 0, j_amr = amr_start[1]; j_loc < nbrCells[1]; ++j_loc, ++j_amr) + { + for (std::uint32_t k_loc = 0, k_amr = amr_start[2]; k_loc < nbrCells[2]; + ++k_loc, ++k_amr) + { + auto amr_point{core::Point{i_amr, j_amr, k_amr}}; + auto amr_index = amr_point.template toArray(); + + std::size_t nbr = 0; + + for (auto& pop : ions) + { + const auto& domainParticles = pop.domainParticles(); + + nbr += domainParticles.nbr_particles_in(amr_index); + } + + lb_view(i_loc, j_loc, k_loc) = nbr; + } + } + } + } + } + + // TODO here, we have the lb_view value correctly set on all patches. we also know the id_ + // so this is where we should call setWorkloadPatchDataIndex... which is a method of the CascadePartitioner + // lb_view is a local container containing the datz + // the loadbalancezrmanager knows the id, as well as the loadbalancerestimator + // and the loadbalancerestimator is a cascadpartotioner + +} + +} // namespace PHARE::amr + +#endif diff --git a/src/amr/load_balancing/load_balancer_estimator.hpp b/src/amr/load_balancing/load_balancer_estimator.hpp new file mode 100644 index 000000000..8b0dc8ef3 --- /dev/null +++ b/src/amr/load_balancing/load_balancer_estimator.hpp @@ -0,0 +1,38 @@ +#ifndef PHARE_LOAD_BALANCER_ESTIMATOR_HPP +#define PHARE_LOAD_BALANCER_ESTIMATOR_HPP + +#include +#include + +#include +#include "SAMRAI/mesh/CascadePartitioner.h" + +#include "amr/resources_manager/amr_utils.hpp" +#include "amr/physical_models/physical_model.hpp" + + + +namespace PHARE::amr +{ +class LoadBalancerEstimator +{ +public: + LoadBalancerEstimator(int const id) + : id_{id} + { + } + + virtual ~LoadBalancerEstimator() = default; + + virtual void estimate(SAMRAI::hier::PatchLevel& level, + PHARE::solver::IPhysicalModel& model) + = 0; + + +protected: + int const id_; +}; + +} // namespace PHARE::amr + +#endif diff --git a/src/amr/load_balancing/load_balancer_estimator_hybrid.hpp b/src/amr/load_balancing/load_balancer_estimator_hybrid.hpp new file mode 100644 index 000000000..dc87cfa9b --- /dev/null +++ b/src/amr/load_balancing/load_balancer_estimator_hybrid.hpp @@ -0,0 +1,174 @@ +#ifndef PHARE_LOAD_BALANCER_ESTIMATOR_HYBRID_HPP +#define PHARE_LOAD_BALANCER_ESTIMATOR_HYBRID_HPP + +#include +#include +#include + +#include "load_balancer_estimator.hpp" +// #include "amr/resources_manager/amr_utils.hpp" +#include "amr/physical_models/physical_model.hpp" +// #include "core/data/ndarray/ndarray_vector.hpp" +// #include "amr/resources_manager/amr_utils.hpp" +// #include "core/utilities/point/point.hpp" +#include "load_balancer_hybrid_strategy.hpp" +#include "load_balancer_hybrid_strategy_factory.hpp" + + + + +namespace PHARE::amr +{ +template +class LoadBalancerEstimatorHybrid : public LoadBalancerEstimator +{ + using HybridModel = typename PHARE_T::HybridModel_t; + using gridlayout_type = typename HybridModel::gridlayout_type; + + + +public: + // LoadBalancerEstimatorHybrid(int const id) + LoadBalancerEstimatorHybrid(std::string strategy_name, int const id) + : LoadBalancerEstimator{id} + , strat_{LoadBalancerHybridStrategyFactory::create(strategy_name, id)} + { + } + + ~LoadBalancerEstimatorHybrid() = default; // the implementation of a virtual class NEEDS a dtor + + void estimate(SAMRAI::hier::PatchLevel& level, + solver::IPhysicalModel& model) override; + + +private: + std::unique_ptr> strat_; +}; + + + +template +inline void +LoadBalancerEstimatorHybrid::estimate(SAMRAI::hier::PatchLevel& level, + solver::IPhysicalModel& model) +{ + strat_->compute(level, model); + + // static auto constexpr dimension = HybridModel::dimension; + // auto& hybridModel = dynamic_cast(model); + // auto& resourcesManager = hybridModel.resourcesManager; + // auto& ions = hybridModel.state.ions; + + // bool constexpr c_ordering = false; + + // for (auto& patch : level) + // { + // auto const& layout = layoutFromPatch(*patch); + + // auto patch_data_lb + // = + // dynamic_cast*>(patch->getPatchData(this->id_).get()); + // auto load_balancer_val = patch_data_lb->getPointer(); + + // const auto& box = patch->getBox(); + + // auto lb_view = core::NdArrayView(load_balancer_val, + // layout.nbrCells()); + + // auto _ = resourcesManager->setOnPatch(*patch, ions); + + // // The view on "load_balancer_val" do not have any ghost cells, meaning that this patch + // // data lies on a box defind by the nbr of cells in the physical domain + // // As the nbr of ghost cells in the box associated to the load balancer patch data is + // // null, we hence loop on an AMR index (then needing to firstly get the AMR box) + // // to get the AMRLocatStart and AMRLocalEnd. + // // Then, we build "by hand" the local index for the "lb_view" considering that the nbr + // // of ghost cell for this patch data is null. + // auto amr_box = layout.AMRBox(); + + // // The lb_view is a CellData, meaning that it is dual as the index of an amr box + // auto amr_start = amr_box.lower; + // auto amr_end = amr_box.upper; + + // // nbr of cells in the physical domain + // auto nbrCells = layout.nbrCells(); + + + // if constexpr (dimension == 1) + // { + // for (auto i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + // { + // auto amr_point{core::Point{i_amr}}; + // auto amr_index = amr_point.template toArray(); + + // auto nbr = 0; + + // for (auto& pop : ions) + // { + // const auto& domainParticles = pop.domainParticles(); + + // nbr += domainParticles.nbr_particles_in(amr_index); + // } + + // lb_view(i_loc) = nbr; + // } + // } + + + // if constexpr (dimension == 2) + // { + // for (auto i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + // { + // for (auto j_loc = 0, j_amr = amr_start[1]; j_loc < nbrCells[1]; ++j_loc, ++j_amr) + // { + // auto amr_point{core::Point{i_amr, j_amr}}; + // auto amr_index = amr_point.template toArray(); + + // auto nbr = 0; + + // for (auto& pop : ions) + // { + // const auto& domainParticles = pop.domainParticles(); + + // nbr += domainParticles.nbr_particles_in(amr_index); + // } + + // lb_view(i_loc, j_loc) = nbr; + // } + // } + // } + + + // if constexpr (dimension == 3) + // { + // for (auto i_loc = 0, i_amr = amr_start[0]; i_loc < nbrCells[0]; ++i_loc, ++i_amr) + // { + // for (auto j_loc = 0, j_amr = amr_start[1]; j_loc < nbrCells[1]; ++j_loc, ++j_amr) + // { + // for (auto k_loc = 0, k_amr = amr_start[2]; k_loc < nbrCells[2]; + // ++k_loc, ++k_amr) + // { + // auto amr_point{core::Point{i_amr, j_amr, k_amr}}; + // auto amr_index = amr_point.template toArray(); + + // auto nbr = 0; + + // for (auto& pop : ions) + // { + // const auto& domainParticles = pop.domainParticles(); + + // nbr += domainParticles.nbr_particles_in(amr_index); + // } + + // lb_view(i_loc, j_loc, k_loc) = nbr; + // } + // } + // } + // } + // } +} + +} // namespace PHARE::amr + +#endif diff --git a/src/amr/load_balancing/load_balancer_hybrid_strategy.hpp b/src/amr/load_balancing/load_balancer_hybrid_strategy.hpp new file mode 100644 index 000000000..59edef854 --- /dev/null +++ b/src/amr/load_balancing/load_balancer_hybrid_strategy.hpp @@ -0,0 +1,26 @@ + +#ifndef PHARE_LOAD_BALANCER_HYBRID_STRATEGY_HPP +#define PHARE_LOAD_BALANCER_HYBRID_STRATEGY_HPP + +#include + +#include "amr/physical_models/physical_model.hpp" +#include "amr/types/amr_types.hpp" + + + +namespace PHARE::amr +{ +template +class LoadBalancerHybridStrategy +{ +public: + virtual void compute(SAMRAI::hier::PatchLevel& level, + PHARE::solver::IPhysicalModel& model) + = 0; +}; + + +} // namespace PHARE::amr + +#endif diff --git a/src/amr/load_balancing/load_balancer_hybrid_strategy_factory.hpp b/src/amr/load_balancing/load_balancer_hybrid_strategy_factory.hpp new file mode 100644 index 000000000..f85a74f09 --- /dev/null +++ b/src/amr/load_balancing/load_balancer_hybrid_strategy_factory.hpp @@ -0,0 +1,38 @@ + +#ifndef LOAD_BALANCER_HYBRID_STRATEGY_FACTORY_HPP +#define LOAD_BALANCER_HYBRID_STRATEGY_FACTORY_HPP + + +#include + +#include "amr/load_balancing/load_balancer_hybrid_strategy.hpp" +#include "amr/load_balancing/concrete_load_balancer_hybrid_strategy_nppc.hpp" + + + +namespace PHARE::amr +{ +template +class LoadBalancerHybridStrategyFactory +{ +public: + static std::unique_ptr> create(std::string strat_name, + int const id) + { + if (strat_name == "nppc") + { + return std::make_unique>(id); + } + + else if (strat_name == "homogeneous") + { + return std::make_unique>(id); + } + + return nullptr; + } +}; + +} // namespace PHARE::amr + +#endif diff --git a/src/amr/load_balancing/load_balancer_manager.hpp b/src/amr/load_balancing/load_balancer_manager.hpp new file mode 100644 index 000000000..d3accc7bc --- /dev/null +++ b/src/amr/load_balancing/load_balancer_manager.hpp @@ -0,0 +1,115 @@ +#ifndef PHARE_LOAD_BALANCER_MANAGER_HPP +#define PHARE_LOAD_BALANCER_MANAGER_HPP + +#include +#include +#include + +#include +#include +// #include "phare_core.hpp" +#include "initializer/data_provider.hpp" +#include "load_balancer_estimator.hpp" +// #include "amr/resources_manager/amr_utils.hpp" +// #include "amr/solvers/solver.hpp" + + + +namespace PHARE::amr +{ +template +class LoadBalancerManager +{ +public: + // LoadBalancerManager(int const maxLevelNumber) + LoadBalancerManager(PHARE::initializer::PHAREDict const& dict) + : dim_{SAMRAI::tbox::Dimension{dim}} + , loadBalancerVar_{std::make_shared>( + dim_, "LoadBalancerVariable")} + , variableDatabase_{SAMRAI::hier::VariableDatabase::getDatabase()} + , context_{variableDatabase_->getContext("default")} + , id_{variableDatabase_->registerVariableAndContext(loadBalancerVar_, context_, + SAMRAI::hier::IntVector::getZero(dim_))} + , maxLevelNumber_{dict["simulation"]["AMR"]["max_nbr_levels"].template to()} + // , loadBalancerEstimators_(maxLevelNumber, nullptr){}; + , loadBalancerEstimators_(maxLevelNumber_, nullptr){}; + + ~LoadBalancerManager() { variableDatabase_->removeVariable("LoadBalancerVariable"); }; + + int getId() const; + + void addLoadBalancerEstimator(int const iLevel_min, int const iLevel_max, + std::shared_ptr lbe); + + void addLoadBalancer(std::unique_ptr loadBalancer) + { + loadBalancer_ = std::move(loadBalancer); + loadBalancer_->setWorkloadPatchDataIndex(id_); + } + + void allocate(SAMRAI::hier::Patch& patch, double const allocateTime); + + void estimate(SAMRAI::hier::PatchLevel& level, + PHARE::solver::IPhysicalModel& model); + + +private: + SAMRAI::tbox::Dimension dim_; + std::shared_ptr> loadBalancerVar_; + SAMRAI::hier::VariableDatabase* variableDatabase_; + std::shared_ptr context_; + int const id_; + int const maxLevelNumber_; + std::vector> loadBalancerEstimators_; + std::unique_ptr loadBalancer_; +}; + + + + +template +inline int LoadBalancerManager::getId() const +{ + return id_; +} + + + + +template +inline void +LoadBalancerManager::addLoadBalancerEstimator(int const iLevel_min, int const iLevel_max, + std::shared_ptr lbe) +{ + for (auto ilevel = iLevel_min; ilevel <= iLevel_max; ilevel++) + { + loadBalancerEstimators_[ilevel] = lbe; + } +} + + + + +template +inline void LoadBalancerManager::allocate(SAMRAI::hier::Patch& patch, + double const allocateTime) +{ + patch.allocatePatchData(id_, allocateTime); +} + + + +template +inline void +LoadBalancerManager::estimate(SAMRAI::hier::PatchLevel& level, + PHARE::solver::IPhysicalModel& model) +{ + auto iLevel = level.getLevelNumber(); + auto lbe = loadBalancerEstimators_[iLevel]; + + lbe->estimate(level, model); +} + +} // namespace PHARE::amr + +#endif diff --git a/src/amr/multiphysics_integrator.hpp b/src/amr/multiphysics_integrator.hpp index 21c463523..6794b0d13 100644 --- a/src/amr/multiphysics_integrator.hpp +++ b/src/amr/multiphysics_integrator.hpp @@ -32,6 +32,8 @@ #include "core/utilities/algorithm.hpp" +#include "load_balancing/load_balancer_manager.hpp" +#include "load_balancing/load_balancer_estimator.hpp" #include "phare_core.hpp" @@ -46,6 +48,7 @@ namespace solver int solverIndex = NOT_SET; int resourcesManagerIndex = NOT_SET; int taggerIndex = NOT_SET; + // int loadBalancerIndex = NOT_SET; std::string messengerName; }; @@ -101,6 +104,7 @@ namespace solver , levelDescriptors_(dict["AMR"]["max_nbr_levels"].template to()) , simFuncs_{simFuncs} , dict_{dict} + , load_balancer_manager_{nullptr} { // auto mhdSolver = std::make_unique>(resourcesManager_); @@ -319,6 +323,7 @@ namespace solver model.allocate(*patch, initDataTime); solver.allocate(model, *patch, initDataTime); messenger.allocate(*patch, initDataTime); + load_balancer_manager_->allocate(*patch, initDataTime); } } @@ -354,14 +359,22 @@ namespace solver int const coarsestLevel, int const finestLevel) override { // handle samrai restarts / schedule creation - // allocation of patch datas which may not want to be saved to restart files will - // likely need to go here somehow https://github.com/PHAREHUB/PHARE/issues/664 + // allocation of patch datas which may not want to be saved to restart files will + // likely need to go here somehow https://github.com/PHAREHUB/PHARE/issues/664 if (!restartInitialized_ and SAMRAI::tbox::RestartManager::getManager()->isFromRestart()) { auto& messenger = getMessengerWithCoarser_(coarsestLevel); for (auto ilvl = coarsestLevel; ilvl <= finestLevel; ++ilvl) + { messenger.registerLevel(hierarchy, ilvl); + auto level = hierarchy->getPatchLevel(ilvl); + for (auto& patch : *level) + { + auto time = dict_["restarts"]["restart_time"].template to(); + load_balancer_manager_->allocate(*patch, time); + } + } restartInitialized_ = true; } } @@ -500,6 +513,8 @@ namespace solver dump_(iLevel); } + load_balancer_manager_->estimate(*level, model); + return newTime; } @@ -554,6 +569,11 @@ namespace solver bool usingRefinedTimestepping() const override { return true; } + void setLoadBalancerManager(std::unique_ptr> lbm) + { + load_balancer_manager_ = std::move(lbm); + } + private: @@ -571,6 +591,7 @@ namespace solver std::map> levelInitializers_; SimFunctors const& simFuncs_; PHARE::initializer::PHAREDict const& dict_; + std::unique_ptr> load_balancer_manager_; bool validLevelRange_(int coarsestLevel, int finestLevel) @@ -583,6 +604,7 @@ namespace solver } + bool existTaggerOnRange_(int coarsestLevel, int finestLevel) { for (auto iLevel = coarsestLevel; iLevel <= finestLevel; ++iLevel) @@ -597,6 +619,19 @@ namespace solver + // bool existloadBalancererOnRange_(int coarsestLevel, int finestLevel) + // { + // for (auto iLevel = coarsestLevel; iLevel <= finestLevel; ++iLevel) + // { + // if (levelDescriptors_[iLevel].loadBalancerIndex != LevelDescriptor::NOT_SET) + // { + // return true; + // } + // } + // return false; + // } + + bool existModelOnRange_(int coarsestLevel, int finestLevel) { diff --git a/src/amr/wrappers/integrator.hpp b/src/amr/wrappers/integrator.hpp index 9efd9ff3e..25455e70d 100644 --- a/src/amr/wrappers/integrator.hpp +++ b/src/amr/wrappers/integrator.hpp @@ -13,7 +13,8 @@ #include #include #include -#include +// #include +#include #include #include #include @@ -70,9 +71,14 @@ Integrator<_dimension>::Integrator( std::shared_ptr tagAndInitStrategy, double startTime, double endTime) { - auto loadBalancer = std::make_shared( + // auto loadBalancer = std::make_shared( + // SAMRAI::tbox::Dimension{dimension}, "LoadBalancer"); + + auto loadBalancer = std::make_shared( SAMRAI::tbox::Dimension{dimension}, "LoadBalancer"); + loadBalancer->setSAMRAI_MPI(SAMRAI::tbox::SAMRAI_MPI::getSAMRAIWorld()); // TODO Is it really needed ? + auto refineDB = getUserRefinementBoxesDatabase(dict["simulation"]["AMR"]); auto standardTag = std::make_shared( "StandardTagAndInitialize", tagAndInitStrategy.get(), refineDB); diff --git a/src/core/data/particles/particle_array.hpp b/src/core/data/particles/particle_array.hpp index 6f2a28a1a..93acc3066 100644 --- a/src/core/data/particles/particle_array.hpp +++ b/src/core/data/particles/particle_array.hpp @@ -152,6 +152,9 @@ class ParticleArray NO_DISCARD auto nbr_particles_in(box_t const& box) const { return cellMap_.size(box); } + using cell_t = std::array; + auto nbr_particles_in(cell_t const& cell) const { return cellMap_.size(cell); } + void export_particles(box_t const& box, ParticleArray& dest) const { PHARE_LOG_SCOPE(3, "ParticleArray::export_particles"); diff --git a/src/simulator/simulator.hpp b/src/simulator/simulator.hpp index 395d02831..2ec834912 100644 --- a/src/simulator/simulator.hpp +++ b/src/simulator/simulator.hpp @@ -1,7 +1,8 @@ #ifndef PHARE_SIMULATOR_SIMULATOR_HPP #define PHARE_SIMULATOR_SIMULATOR_HPP - +#include +#include #include "phare_core.hpp" #include "phare_types.hpp" @@ -12,10 +13,8 @@ #include "core/utilities/mpi_utils.hpp" #include "core/utilities/timestamps.hpp" #include "amr/tagging/tagger_factory.hpp" - -#include -#include -#include +#include "amr/load_balancing/load_balancer_manager.hpp" +#include "amr/load_balancing/load_balancer_estimator_hybrid.hpp" namespace PHARE @@ -262,6 +261,22 @@ void Simulator::hybrid_init(initializer::PHAREDict auto hybridTagger_ = amr::TaggerFactory::make("HybridModel", "default"); multiphysInteg_->registerTagger(0, maxLevelNumber_ - 1, std::move(hybridTagger_)); + + + + auto lbm_ = std::make_unique>(dict); + + auto lbe_ = std::make_unique>( + dict["simulation"]["AMR"]["loadbalancing"].template to(), lbm_->getId()); + + lbm_->addLoadBalancerEstimator(0, maxLevelNumber_ - 1, std::move(lbe_)); + lbm_->addLoadBalancer(std::make_unique( + SAMRAI::tbox::Dimension{dim}, "cascade")); + multiphysInteg_->setLoadBalancerManager(std::move(lbm_)); + + + + if (dict["simulation"].contains("restarts")) startTime_ = restarts_init(dict["simulation"]["restarts"]);