load balancing L0 by particle count

pyphare/pyphare/pharein/__init__.py

@@ -78,9 +78,7 @@ def __init__(self, fn):
         self.fn = fn
 
     def __call__(self, *xyz):
-        args = []
-        for i, arg in enumerate(xyz):
-            args.append(np.asarray(arg))
+        args = [np.asarray(arg) for arg in xyz]
         ret = self.fn(*args)
         if isinstance(ret, list):
             ret = np.asarray(ret)
@@ -215,6 +213,15 @@ def as_paths(rb):
     add_double("simulation/algo/ohm/resistivity", simulation.resistivity)
     add_double("simulation/algo/ohm/hyper_resistivity", simulation.hyper_resistivity)
 
+    for k, v in simulation.advanced.items():
+        path = f"simulation/advanced/{k}"
+        if isinstance(v, int):
+            add_int(path, v)
+        elif isinstance(v, float):
+            add_double(path, v)
+        else:
+            add_string(path, v)
+
     init_model = simulation.model
     modelDict = init_model.model_dict
 
@@ -238,12 +245,20 @@ def as_paths(rb):
         addInitFunction(partinit_path + "thermal_velocity_x", fn_wrapper(d["vthx"]))
         addInitFunction(partinit_path + "thermal_velocity_y", fn_wrapper(d["vthy"]))
         addInitFunction(partinit_path + "thermal_velocity_z", fn_wrapper(d["vthz"]))
-        add_int(partinit_path + "nbr_part_per_cell", d["nbrParticlesPerCell"])
         add_double(partinit_path + "charge", d["charge"])
         add_string(partinit_path + "basis", "cartesian")
         if "init" in d and "seed" in d["init"]:
             pp.add_optional_size_t(partinit_path + "init/seed", d["init"]["seed"])
 
+        if isinstance(d["nbrParticlesPerCell"], tuple):
+            addInitFunction(
+                partinit_path + "nbr_part_per_cell_fn",
+                fn_wrapper(d["nbrParticlesPerCell"][0]),
+            )
+            add_int(partinit_path + "nbr_part_per_cell", d["nbrParticlesPerCell"][1])
+        else:
+            add_int(partinit_path + "nbr_part_per_cell", d["nbrParticlesPerCell"])
+
     add_string("simulation/electromag/name", "EM")
     add_string("simulation/electromag/electric/name", "E")
 

pyphare/pyphare/pharein/simulation.py

@@ -121,6 +121,8 @@ def check_time(**kwargs):
         and "time_step" not in kwargs
     )
 
+    start_time = kwargs.get("restart_options", {}).get("restart_time", 0)
+
     if final_and_dt:
         time_step_nbr = int(kwargs["final_time"] / kwargs["time_step"])
         time_step = kwargs["final_time"] / time_step_nbr
@@ -139,7 +141,11 @@ def check_time(**kwargs):
             + " or 'final_time' and 'time_step_nbr'"
         )
 
-    return time_step_nbr, time_step, kwargs.get("final_time", time_step * time_step_nbr)
+    return (
+        time_step_nbr,
+        time_step,
+        kwargs.get("final_time", start_time + time_step * time_step_nbr),
+    )
 
 
 # ------------------------------------------------------------------------------
@@ -616,6 +622,7 @@ def wrapper(simulation_object, **kwargs):
             "tag_buffer",
             "description",
             "loadbalancing",
+            "advanced",
         ]
 
         accepted_keywords += check_optional_keywords(**kwargs)
@@ -684,6 +691,8 @@ def wrapper(simulation_object, **kwargs):
 
         kwargs["hyper_resistivity"] = check_hyper_resistivity(**kwargs)
 
+        kwargs["advanced"] = kwargs.get("advanced", {})
+
         return func(simulation_object, **kwargs)
 
     return wrapper
@@ -846,9 +855,6 @@ def __init__(self, **kwargs):
         ]
         validate_restart_options(self)
 
-    def final_time(self):
-        return self.time_step * self.time_step_nbr
-
     def simulation_domain(self):
         return [dl * n + ori for dl, n, ori in zip(self.dl, self.cells, self.origin)]
 

pyphare/pyphare/pharesee/hierarchy.py

@@ -595,7 +595,7 @@ def __init__(
                     for ilvl, level in levels.items():
                         patches = []
                         for patch in level.patches:
-                            patches += [Patch({qty: patch.patch_datas[qty]})]
+                            patches += [Patch({qty: patch.patch_datas[qty]}, patch.id)]
                         new_lvls[ilvl] = PatchLevel(ilvl, patches)
                     if first:
                         self.__dict__[qty] = PatchHierarchy(
@@ -1283,7 +1283,9 @@ def new_patches_from(compute, hierarchies, ilvl, **kwargs):
         current_patch = reference_hier.patch_levels[ilvl].patches[ip]
         layout = current_patch.layout
         patch_datas = extract_patchdatas(hierarchies, ilvl, ip)
-        new_patch_datas = new_patchdatas_from(compute, patch_datas, layout, **kwargs)
+        new_patch_datas = new_patchdatas_from(
+            compute, patch_datas, layout, id=current_patch.id, **kwargs
+        )
         new_patches.append(Patch(new_patch_datas, current_patch.id))
     return new_patches
 

pyphare/pyphare/simulator/simulator.py

@@ -1,3 +1,4 @@
+import datetime
 import atexit
 import time as timem
 import numpy as np
@@ -29,6 +30,29 @@ def startMPI():
         life_cycles["samrai"] = cpp_lib().SamraiLifeCycle()
 
 
+def print_rank0(*args, **kwargs):
+    from pyphare.cpp import cpp_lib
+
+    if cpp_lib().mpi_rank() == 0:
+        print(*args, **kwargs)
+    cpp_lib().mpi_barrier()
+
+
+def plot_timestep_time(timestep_times):
+    from pyphare.cpp import cpp_lib
+
+    if cpp_lib().mpi_rank() == 0:
+        import matplotlib.pyplot as plt
+
+        fig, ax = plt.subplots()
+        ax.plot(timestep_times)
+        plt.ylabel("timestep time")
+        plt.xlabel("timestep")
+        fig.savefig("timestep_times.png")
+
+    cpp_lib().mpi_barrier()
+
+
 class Simulator:
     def __init__(self, simulation, auto_dump=True, **kwargs):
         import pyphare.pharein as ph
@@ -107,8 +131,7 @@ def _throw(self, e):
         import sys
         from pyphare.cpp import cpp_lib
 
-        if cpp_lib().mpi_rank() == 0:
-            print(e)
+        print_rank0(e)
         sys.exit(1)
 
     def advance(self, dt=None):
@@ -134,7 +157,7 @@ def times(self):
             self.timeStep(),
         )
 
-    def run(self):
+    def run(self, plot_times=False):
         from pyphare.cpp import cpp_lib
 
         self._check_init()
@@ -153,8 +176,11 @@ def run(self):
                 out = f"t = {t:8.5f}  -  {ticktock:6.5f}sec  - total {np.sum(perf):7.4}sec"
                 print(out, end=self.print_eol)
 
-        print("mean advance time = {}".format(np.mean(perf)))
-        print("total advance time = {}".format(np.sum(perf)))
+        print_rank0(f"mean advance time = {np.mean(perf)}")
+        print_rank0(f"total advance time = {datetime.timedelta(seconds=np.sum(perf))}")
+
+        if plot_times:
+            plot_timestep_time(perf)
 
         return self.reset()
 

res/cmake/dep.cmake

@@ -1,4 +1,6 @@
 
+include("${PHARE_PROJECT_DIR}/res/cmake/dep/cppdict.cmake")
+add_subdirectory(subprojects/cppdict)
 
 # SAMRAI build option
 include("${PHARE_PROJECT_DIR}/res/cmake/dep/samrai.cmake")
@@ -15,6 +17,3 @@ include("${PHARE_PROJECT_DIR}/res/cmake/dep/pybind.cmake")
 
 # HighFive
 include("${PHARE_PROJECT_DIR}/res/cmake/dep/highfive.cmake")
-
-
-add_subdirectory(subprojects/cppdict)

res/cmake/dep/caliper.cmake

@@ -2,6 +2,8 @@
 #  TO activate caliper with PHARE
 #   1. configure cmake -DwithCaliper=ON
 #   2. export CALI_CONFIG=runtime-report
+#   or for inclusive timings per function (makes more sense IMO)
+#   3. export CALI_CONFIG=runtime-report,calc.inclusive
 ##
 
 if(DEFINED CALIPER_ROOT)

res/cmake/dep/cppdict.cmake

@@ -0,0 +1,14 @@
+
+
+set(CPPDICT_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/subprojects/cppdict)
+
+if (NOT EXISTS ${CPPDICT_SRCDIR})
+  execute_process(
+    COMMAND ${Git} clone https://github.com/LaboratoryOfPlasmaPhysics/cppdict ${CPPDICT_SRCDIR} -b master --recursive --depth 10
+  )
+else()
+  if(devMode)
+    message("downloading latest cppdict updates")
+    execute_process(COMMAND ${Git} pull origin master WORKING_DIRECTORY ${CPPDICT_SRCDIR})
+  endif(devMode)
+endif()

src/amr/level_initializer/hybrid_level_initializer.hpp

@@ -52,24 +52,24 @@ namespace solver
             auto& hybridModel = static_cast<HybridModel&>(model);
             auto& level       = amr_types::getLevel(*hierarchy, levelNumber);
 
-            auto& hybMessenger = dynamic_cast<HybridMessenger&>(messenger);
+            auto& hybMessenger  = dynamic_cast<HybridMessenger&>(messenger);
+            bool isRegriddingL0 = isRegridding and levelNumber == 0;
 
-            if (isRootLevel(levelNumber))
+            if (isRegridding)
             {
-                PHARE_LOG_START("hybridLevelInitializer::initialize : root level init");
-                model.initialize(level);
-                messenger.fillRootGhosts(model, level, initDataTime);
-                PHARE_LOG_STOP("hybridLevelInitializer::initialize : root level init");
+                std::cout << "regriding level " << levelNumber << "\n";
+                PHARE_LOG_START("hybridLevelInitializer::initialize : regriding block");
+                messenger.regrid(hierarchy, levelNumber, oldLevel, model, initDataTime);
+                PHARE_LOG_STOP("hybridLevelInitializer::initialize : regriding block");
             }
-
             else
             {
-                if (isRegridding)
+                if (isRootLevel(levelNumber))
                 {
-                    std::cout << "regriding level " << levelNumber << "\n";
-                    PHARE_LOG_START("hybridLevelInitializer::initialize : regriding block");
-                    messenger.regrid(hierarchy, levelNumber, oldLevel, model, initDataTime);
-                    PHARE_LOG_STOP("hybridLevelInitializer::initialize : regriding block");
+                    PHARE_LOG_START("hybridLevelInitializer::initialize : root level init");
+                    model.initialize(level);
+                    messenger.fillRootGhosts(model, level, initDataTime);
+                    PHARE_LOG_STOP("hybridLevelInitializer::initialize : root level init");
                 }
                 else
                 {
@@ -109,55 +109,59 @@ namespace solver
             // we are at a sync time across levels and that the time interpolation
             // is not needed. But is still seems to use the messenger tempoeraries like
             // NiOld etc. so prepareStep() must be called, see end of the function.
-            hybMessenger.fillIonMomentGhosts(hybridModel.state.ions, level, initDataTime);
+
+            if (!isRegriddingL0)
+                hybMessenger.fillIonMomentGhosts(hybridModel.state.ions, level, initDataTime);
 
 
             // now moments are known everywhere, compute J and E
             // via Ampere and Ohm
             // this only needs to be done for the root level
             // since otherwise initLevel has done it already
-
-            if (isRootLevel(levelNumber))
-            {
-                auto& B = hybridModel.state.electromag.B;
-                auto& J = hybridModel.state.J;
-
-                for (auto& patch : level)
-                {
-                    auto _      = hybridModel.resourcesManager->setOnPatch(*patch, B, J);
-                    auto layout = PHARE::amr::layoutFromPatch<GridLayoutT>(*patch);
-                    auto __     = core::SetLayout(&layout, ampere_);
-                    ampere_(B, J);
-
-                    hybridModel.resourcesManager->setTime(J, *patch, 0.);
-                }
-                hybMessenger.fillCurrentGhosts(J, levelNumber, 0.);
-
-                auto& electrons = hybridModel.state.electrons;
-                auto& E         = hybridModel.state.electromag.E;
-
-                for (auto& patch : level)
+            if (!isRegriddingL0)
+                if (isRootLevel(levelNumber))
                 {
-                    auto layout = PHARE::amr::layoutFromPatch<GridLayoutT>(*patch);
-                    auto _ = hybridModel.resourcesManager->setOnPatch(*patch, B, E, J, electrons);
-                    electrons.update(layout);
-                    auto& Ve = electrons.velocity();
-                    auto& Ne = electrons.density();
-                    auto& Pe = electrons.pressure();
-                    auto __  = core::SetLayout(&layout, ohm_);
-                    ohm_(Ne, Ve, Pe, B, J, E);
-                    hybridModel.resourcesManager->setTime(E, *patch, 0.);
+                    auto& B = hybridModel.state.electromag.B;
+                    auto& J = hybridModel.state.J;
+
+                    for (auto& patch : level)
+                    {
+                        auto _      = hybridModel.resourcesManager->setOnPatch(*patch, B, J);
+                        auto layout = PHARE::amr::layoutFromPatch<GridLayoutT>(*patch);
+                        auto __     = core::SetLayout(&layout, ampere_);
+                        ampere_(B, J);
+
+                        hybridModel.resourcesManager->setTime(J, *patch, 0.);
+                    }
+                    hybMessenger.fillCurrentGhosts(J, levelNumber, 0.);
+
+                    auto& electrons = hybridModel.state.electrons;
+                    auto& E         = hybridModel.state.electromag.E;
+
+                    for (auto& patch : level)
+                    {
+                        auto layout = PHARE::amr::layoutFromPatch<GridLayoutT>(*patch);
+                        auto _
+                            = hybridModel.resourcesManager->setOnPatch(*patch, B, E, J, electrons);
+                        electrons.update(layout);
+                        auto& Ve = electrons.velocity();
+                        auto& Ne = electrons.density();
+                        auto& Pe = electrons.pressure();
+                        auto __  = core::SetLayout(&layout, ohm_);
+                        ohm_(Ne, Ve, Pe, B, J, E);
+                        hybridModel.resourcesManager->setTime(E, *patch, 0.);
+                    }
+
+                    hybMessenger.fillElectricGhosts(E, levelNumber, 0.);
                 }
 
-                hybMessenger.fillElectricGhosts(E, levelNumber, 0.);
-            }
-
             // quantities have been computed on the level,like the moments and J
             // that we later in the code need to get on level ghost nodes via
             // space and TIME interpolation. We thus need to save current values
             // in "old" messenger temporaries.
             // NOTE :  this may probably be skipped for finest level since, TBC at some point
-            hybMessenger.prepareStep(hybridModel, level, initDataTime);
+            if (!isRegriddingL0)
+                hybMessenger.prepareStep(hybridModel, level, initDataTime);
         }
     };
 } // namespace solver