diff --git a/pyphare/pyphare/pharein/__init__.py b/pyphare/pyphare/pharein/__init__.py index 9ff8f7d57..11f4ac1e8 100644 --- a/pyphare/pyphare/pharein/__init__.py +++ b/pyphare/pyphare/pharein/__init__.py @@ -43,6 +43,7 @@ serialize as serialize_sim, deserialize as deserialize_sim, ) +from .load_balancer import LoadBalancer def getSimulation(): @@ -118,6 +119,9 @@ def populateDict(): def add_int(path, val): pp.add_int(path, int(val)) + def add_bool(path, val): + pp.add_bool(path, bool(val)) + def add_double(path, val): pp.add_double(path, float(val)) @@ -172,8 +176,6 @@ def add_vector_int(path, val): add_int("simulation/AMR/tag_buffer", simulation.tag_buffer) - add_string("simulation/AMR/loadbalancing", simulation.loadbalancing) - refinement_boxes = simulation.refinement_boxes def as_paths(rb): @@ -213,14 +215,16 @@ def as_paths(rb): add_double("simulation/algo/ohm/resistivity", simulation.resistivity) add_double("simulation/algo/ohm/hyper_resistivity", simulation.hyper_resistivity) - for k, v in simulation.advanced.items(): - path = f"simulation/advanced/{k}" - if isinstance(v, int): - add_int(path, v) - elif isinstance(v, float): - add_double(path, v) - else: - add_string(path, v) + # load balancer block start + lb = simulation.load_balancer or LoadBalancer(_register=False) + base = "simulation/AMR/loadbalancing" + add_bool(f"{base}/auto", lb.auto) + add_bool(f"{base}/active", lb.active) + add_bool(f"{base}/on_init", lb.on_init) + add_size_t(f"{base}/every", lb.every) + add_string(f"{base}/mode", lb.mode) + add_double(f"{base}/tolerance", lb.tol) + # load balancer block end init_model = simulation.model modelDict = init_model.model_dict diff --git a/pyphare/pyphare/pharein/load_balancer.py b/pyphare/pyphare/pharein/load_balancer.py new file mode 100644 index 000000000..8cc9262aa --- /dev/null +++ b/pyphare/pyphare/pharein/load_balancer.py @@ -0,0 +1,45 @@ +# +# + +from dataclasses import dataclass, field +from . import global_vars as gv + + +@dataclass +class LoadBalancer: + # whether or not load balancing is performed + active: bool = field(default_factory=lambda: False) + + # which way load is assessed + mode: str = field(default_factory=lambda: "nppc") + + # acceptable imbalance essentially + tol: float = field(default_factory=lambda: 0.05) + + # if auto, other values are not used if active + auto: bool = field(default_factory=lambda: True) + + # if !auto these values are used if active + on_init: bool = field(default_factory=lambda: True) + every: int = field(default_factory=lambda: 1) + + # internal, allows not registering object for default init + _register: bool = field(default_factory=lambda: True) + + def __post_init__(self): + allowed_modes = [ + "nppc", # count particles per rank + "homogeneous", # count cells per rank + ] + + if self.mode not in allowed_modes: + raise RuntimeError(f"LoadBalancer mode '{self.mode}' is not valid") + + if self._register: + if not gv.sim: + raise RuntimeError( + f"LoadBalancer cannot be registered as no simulation exists" + ) + if gv.sim.load_balancer: + raise RuntimeError(f"LoadBalancer is already registered to simulation") + gv.sim.load_balancer = self diff --git a/pyphare/pyphare/pharein/simulation.py b/pyphare/pyphare/pharein/simulation.py index 0a8448810..83614f922 100644 --- a/pyphare/pyphare/pharein/simulation.py +++ b/pyphare/pyphare/pharein/simulation.py @@ -840,6 +840,7 @@ def __init__(self, **kwargs): self.diagnostics = {} self.model = None self.electrons = None + self.load_balancer = None # hard coded in C++ MultiPhysicsIntegrator::getMaxFinerLevelDt self.nSubcycles = 4 diff --git a/src/amr/load_balancing/load_balancer_details.hpp b/src/amr/load_balancing/load_balancer_details.hpp new file mode 100644 index 000000000..42cee4b96 --- /dev/null +++ b/src/amr/load_balancing/load_balancer_details.hpp @@ -0,0 +1,36 @@ +#ifndef PHARE_AMR_LOAD_BALANCER_LOAD_BALANCER_DETAILS_HPP +#define PHARE_AMR_LOAD_BALANCER_LOAD_BALANCER_DETAILS_HPP + +#include +#include + +#include "core/logger.hpp" +#include "initializer/data_provider.hpp" + +namespace PHARE::amr +{ +struct LoadBalancerDetails +{ + bool const active = false; + bool const automatic = false; + bool const on_init = false; + + std::size_t const every = 0; + std::string const mode; + + double const tolerance = .05; + + LoadBalancerDetails static FROM(initializer::PHAREDict const& dict) + { + return {cppdict::get_value(dict, "active", false), + cppdict::get_value(dict, "auto", false), + cppdict::get_value(dict, "on_init", false), + cppdict::get_value(dict, "every", std::size_t{0}), + cppdict::get_value(dict, "mode", std::string{"nppc"}), + cppdict::get_value(dict, "tol", 0.05)}; + } +}; + +} // namespace PHARE::amr + +#endif /* PHARE_AMR_LOAD_BALANCER_LOAD_BALANCER_DETAILS_HPP */ diff --git a/src/amr/wrappers/integrator.hpp b/src/amr/wrappers/integrator.hpp index 510214b87..eaa8f758a 100644 --- a/src/amr/wrappers/integrator.hpp +++ b/src/amr/wrappers/integrator.hpp @@ -24,36 +24,15 @@ #include "initializer/data_provider.hpp" +#include "amr/load_balancing/load_balancer_details.hpp" + namespace PHARE::amr { + template class Integrator { - int static constexpr rebalance_coarsest_every_default = 1000; - - bool static _rebalance_coarsest(initializer::PHAREDict const& dict) - { - return cppdict::get_value(dict, "simulation/advanced/integrator/rebalance_coarsest", 0) > 0; - } - - bool static _rebalance_coarsest_on_init(initializer::PHAREDict const& dict) - { - return cppdict::get_value(dict, "simulation/advanced/integrator/rebalance_coarsest_on_init", - 0) - > 0; - } - - std::size_t static _rebalance_coarsest_every(initializer::PHAREDict const& dict) - { - auto in - = cppdict::get_value(dict, "simulation/advanced/integrator/rebalance_coarsest_every", - rebalance_coarsest_every_default); - if (in < 0) - throw std::runtime_error("rebalance_coarsest_every must be positive"); - return static_cast(in); - } - bool static _is_tagging_refinement(initializer::PHAREDict const& dict) { return cppdict::get_value(dict, "simulation/AMR/refinement/tagging/method", @@ -61,11 +40,6 @@ class Integrator == std::string{"auto"}; } - bool static _rebalance_coarsest_auto(initializer::PHAREDict const& dict) - { - return cppdict::get_value(dict, "simulation/advanced/integrator/rebalance_coarsest_auto", 0) - > 0; - } public: static constexpr std::size_t dimension = _dimension; @@ -74,11 +48,6 @@ class Integrator { bool rebalance_coarsest_now = _should_rebalance_now(); - PHARE_LOG_LINE_STR(is_tagging_refinement - << " " << time_step_idx << " " << rebalance_coarsest << " " - << rebalance_coarsest_on_init << " " << rebalance_coarsest_every << " " - << rebalance_coarsest_now); - auto new_time = timeRefIntegrator_->advanceHierarchy(dt, rebalance_coarsest_now); ++time_step_idx; return new_time; @@ -91,18 +60,15 @@ class Integrator std::shared_ptr timeRefLevelStrategy, std::shared_ptr tagAndInitStrategy, std::shared_ptr loadBalancer, // - double startTime, double endTime, int loadBalancerPatchId); + double startTime, double endTime, amr::LoadBalancerDetails const& lb_info, + int loadBalancerPatchId); private: + amr::LoadBalancerDetails const lb_info_; bool const is_tagging_refinement = false; - bool const rebalance_coarsest = false; - bool const rebalance_coarsest_on_init = false; - bool const rebalance_coarsest_auto = false; + int loadBalancerPatchId_ = -1; std::size_t rebalance_coarsest_auto_back_off = 0; std::size_t time_step_idx = 0; - std::size_t const rebalance_coarsest_every = rebalance_coarsest_every_default; - int loadBalancerPatchId_ = -1; - double loadTolerance_ = .05; std::size_t rebalance_coarsest_auto_back_off_by = 1; @@ -123,40 +89,7 @@ class Integrator return load; } - bool _should_rebalance_now() - { - if (is_tagging_refinement and rebalance_coarsest) - { - if (rebalance_coarsest_auto) - { - if (rebalance_coarsest_auto_back_off == 0) - { - auto workLoads = core::mpi::collect(computeNonUniformWorkLoadForLevel0()); - - auto max_value = *std::max_element(workLoads.begin(), workLoads.end()); - for (auto& workload : workLoads) - workload /= max_value; - auto min_value = *std::min_element(workLoads.begin(), workLoads.end()); - if ((1 - min_value) > loadTolerance_) - { - rebalance_coarsest_auto_back_off_by = 8; - rebalance_coarsest_auto_back_off = rebalance_coarsest_auto_back_off_by; - return true; - } - - rebalance_coarsest_auto_back_off_by *= 2; - rebalance_coarsest_auto_back_off = rebalance_coarsest_auto_back_off_by; - } - else - --rebalance_coarsest_auto_back_off; - } - else // maybe redundant with above calculations - return ((time_step_idx == 0 and rebalance_coarsest_on_init) - or (time_step_idx > 0 and rebalance_coarsest_every > 0 - and time_step_idx % rebalance_coarsest_every == 0)); - } - return false; - } + bool _should_rebalance_now(); }; @@ -180,16 +113,11 @@ Integrator<_dimension>::Integrator( std::shared_ptr hierarchy, std::shared_ptr timeRefLevelStrategy, std::shared_ptr tagAndInitStrategy, - std::shared_ptr loadBalancer, // - double startTime, double endTime, int loadBalancerPatchId) - : is_tagging_refinement{_is_tagging_refinement(dict)} - , rebalance_coarsest{_rebalance_coarsest(dict)} - , rebalance_coarsest_on_init{_rebalance_coarsest_on_init(dict)} - , rebalance_coarsest_auto{_rebalance_coarsest_auto(dict)} - , rebalance_coarsest_every{_rebalance_coarsest_every(dict)} + std::shared_ptr loadBalancer, double startTime, + double endTime, amr::LoadBalancerDetails const& lb_info, int loadBalancerPatchId) + : lb_info_{lb_info} + , is_tagging_refinement{_is_tagging_refinement(dict)} , loadBalancerPatchId_{loadBalancerPatchId} - , loadTolerance_{ - cppdict::get_value(dict, "simulation/advanced/integrator/flexible_load_tolerance", .05)} { loadBalancer->setSAMRAI_MPI( SAMRAI::tbox::SAMRAI_MPI::getSAMRAIWorld()); // TODO Is it really needed ? @@ -240,8 +168,41 @@ Integrator<_dimension>::Integrator( "TimeRefinementIntegrator", db, hierarchy, timeRefLevelStrategy, gridding); } +template +bool Integrator<_dimension>::_should_rebalance_now() +{ + if (is_tagging_refinement and lb_info_.active) + { + if (lb_info_.automatic) + { + if (rebalance_coarsest_auto_back_off == 0) + { + auto workLoads = core::mpi::collect(computeNonUniformWorkLoadForLevel0()); + auto max_value = *std::max_element(workLoads.begin(), workLoads.end()); + for (auto& workload : workLoads) + workload /= max_value; + auto min_value = *std::min_element(workLoads.begin(), workLoads.end()); + if ((1 - min_value) > lb_info_.tolerance) + { + rebalance_coarsest_auto_back_off_by = 8; // todo decide final approach + rebalance_coarsest_auto_back_off = rebalance_coarsest_auto_back_off_by; + return true; + } + rebalance_coarsest_auto_back_off_by *= 2; + rebalance_coarsest_auto_back_off = rebalance_coarsest_auto_back_off_by; + } + else + --rebalance_coarsest_auto_back_off; + } + else + return ((time_step_idx == 0 and lb_info_.on_init) + or (time_step_idx > 0 and lb_info_.every > 0 + and time_step_idx % lb_info_.every == 0)); + } + return false; +} template std::shared_ptr diff --git a/src/initializer/data_provider.hpp b/src/initializer/data_provider.hpp index 24ce7a0f5..c48f31a6e 100644 --- a/src/initializer/data_provider.hpp +++ b/src/initializer/data_provider.hpp @@ -48,9 +48,9 @@ namespace initializer using InitFunction = typename InitFunctionHelper::type; - using PHAREDict = cppdict::Dict, double, std::vector, std::size_t, - std::optional, std::string, InitFunction<1>, - InitFunction<2>, InitFunction<3>>; + using PHAREDict = cppdict::Dict, double, std::vector, + std::size_t, std::optional, std::string, + InitFunction<1>, InitFunction<2>, InitFunction<3>>; class PHAREDictHandler diff --git a/src/initializer/dictator.cpp b/src/initializer/dictator.cpp index 21c4d7b37..5d41fa7e8 100644 --- a/src/initializer/dictator.cpp +++ b/src/initializer/dictator.cpp @@ -45,6 +45,7 @@ PYBIND11_MODULE(dictator, m) m.def("add_size_t", add, "add_size_t"); m.def("add_optional_size_t", add>, "add_optional_size_t"); + m.def("add_bool", add, "add"); m.def("add_int", add, "add"); m.def("add_vector_int", add>, "add"); m.def("add_double", add, "add"); diff --git a/src/simulator/simulator.hpp b/src/simulator/simulator.hpp index 018942afd..cd44d8788 100644 --- a/src/simulator/simulator.hpp +++ b/src/simulator/simulator.hpp @@ -13,10 +13,10 @@ #include "core/utilities/mpi_utils.hpp" #include "core/utilities/timestamps.hpp" #include "amr/tagging/tagger_factory.hpp" +#include "amr/load_balancing/load_balancer_details.hpp" #include "amr/load_balancing/load_balancer_manager.hpp" #include "amr/load_balancing/load_balancer_estimator_hybrid.hpp" - namespace PHARE { class ISimulator @@ -261,20 +261,18 @@ void Simulator::hybrid_init(initializer::PHAREDict auto hybridTagger_ = amr::TaggerFactory::make("HybridModel", "default"); multiphysInteg_->registerTagger(0, maxLevelNumber_ - 1, std::move(hybridTagger_)); + amr::LoadBalancerDetails lb_info + = amr::LoadBalancerDetails::FROM(dict["simulation"]["AMR"]["loadbalancing"]); auto lbm_ = std::make_unique>(dict); - auto lbe_ = std::make_shared>( - dict["simulation"]["AMR"]["loadbalancing"].template to(), lbm_->getId()); + auto lbe_ = std::make_shared>(lb_info.mode, + lbm_->getId()); auto loadBalancer_db = std::make_shared("LoadBalancerDB"); - double flexible_load_tolerance - = cppdict::get_value(dict, "simulation/advanced/integrator/flexible_load_tolerance", .05); - - loadBalancer_db->putDouble("flexible_load_tolerance", flexible_load_tolerance); + loadBalancer_db->putDouble("flexible_load_tolerance", lb_info.tolerance); auto loadBalancer = std::make_shared( SAMRAI::tbox::Dimension{dimension}, "LoadBalancer", loadBalancer_db); - PHARE_LOG_LINE_STR(dict["simulation"]["AMR"]["refinement"].contains("tagging")); if (dict["simulation"]["AMR"]["refinement"].contains("tagging")) { // Load balancers break with refinement boxes - only tagging supported /* @@ -294,8 +292,9 @@ void Simulator::hybrid_init(initializer::PHAREDict if (dict["simulation"].contains("restarts")) startTime_ = restarts_init(dict["simulation"]["restarts"]); - integrator_ = std::make_unique(dict, hierarchy_, multiphysInteg_, multiphysInteg_, - loadBalancer, startTime_, finalTime_, lbm_id); + integrator_ + = std::make_unique(dict, hierarchy_, multiphysInteg_, multiphysInteg_, + loadBalancer, startTime_, finalTime_, lb_info, lbm_id); timeStamper = core::TimeStamperFactory::create(dict["simulation"]); diff --git a/tests/simulator/test_load_balancing.py b/tests/simulator/test_load_balancing.py index 074c1c9c0..9dca5061d 100644 --- a/tests/simulator/test_load_balancing.py +++ b/tests/simulator/test_load_balancing.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +# basically a harris run import unittest import pyphare.pharein as ph @@ -27,7 +28,7 @@ timestamps = [x * time_step for x in range(time_step_nbr + 1)] -def config(diag_dir, advanced={}): +def config(diag_dir, loadbalancing={}): sim = ph.Simulation( time_step_nbr=time_step_nbr, time_step=time_step, @@ -41,8 +42,6 @@ def config(diag_dir, advanced={}): "format": "phareh5", "options": {"dir": diag_dir, "mode": "overwrite"}, }, - advanced=advanced, - loadbalancing="nppc", ) def ppc_by_icell(x, y): @@ -140,6 +139,10 @@ def vthxyz(x, y): ph.ParticleDiagnostics( quantity="domain", write_timestamps=timestamps, population_name="protons" ) + + if loadbalancing: + ph.LoadBalancer(**loadbalancing) + return sim @@ -176,14 +179,7 @@ def test_has_balanced_auto(self): return diag_dir = self.unique_diag_dir_for_test_case(diag_outputs, ndim, interp) - sim = config( - diag_dir, - { - "integrator/rebalance_coarsest": 1, - "integrator/rebalance_coarsest_auto": 1, - "integrator/flexible_load_tolerance": 0.05, - }, - ) + sim = config(diag_dir, dict(active=True, auto=True, mode="nppc", tol=0.05)) self.register_diag_dir_for_cleanup(diag_dir) Simulator(sim).run() @@ -200,13 +196,7 @@ def test_has_balanced_manual(self): diag_dir = self.unique_diag_dir_for_test_case(diag_outputs, ndim, interp) sim = config( - diag_dir, - { - "integrator/rebalance_coarsest": 1, - "integrator/rebalance_coarsest_every": 1, - "integrator/rebalance_coarsest_on_init": 1, - "integrator/flexible_load_tolerance": 0.05, - }, + diag_dir, dict(active=True, on_init=True, every=1, mode="nppc", tol=0.05) ) self.register_diag_dir_for_cleanup(diag_dir) Simulator(sim).run()