diff --git a/.gitignore b/.gitignore index b270b8a47..7d3e76060 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ tools/cmake.sh perf.* **/*.h5 .vscode - +.phare* +REPORT_INFO.zip diff --git a/ISSUES.TXT b/ISSUES.TXT index 175018faa..d4e3d3a9d 100644 --- a/ISSUES.TXT +++ b/ISSUES.TXT @@ -1,5 +1,20 @@ +# Having Issues + +Please run: + +> ./tools/report.sh # or "python3 tools/report.py" with the correct PYTHONPATH + +This will build a zip archive "REPORT_INFO.zip", which you should either email to us, or +Log an issue on github via https://github.com/PHAREHUB/PHARE/issues/new +Outline the context of your issue, and upload the zip + + + + +# Known Issues + 1. OMPI symbol resolution with python3. Affects: OMPI versions < 3 Source: https://github.com/open-mpi/ompi/issues/3705 @@ -10,4 +25,5 @@ 2. Python launching, with C++ MPI init, doesn't work so well with MPI4PY Source: https://bitbucket.org/mpi4py/mpi4py/issues/154/attempting-to-use-an-mpi-routine-before Solution: - LD_PRELOAD=/path/to/your/libmpi.so python3 $SCRIPT \ No newline at end of file + LD_PRELOAD=/path/to/your/libmpi.so python3 $SCRIPT + diff --git a/pyphare/pyphare/core/__init__.py b/pyphare/pyphare/core/__init__.py index e69de29bb..446256da2 100644 --- a/pyphare/pyphare/core/__init__.py +++ b/pyphare/pyphare/core/__init__.py @@ -0,0 +1,26 @@ +# +# +# +# program help looks like +""" +usage: phare_sim.py [-h] [-d] + +options: + -h, --help show this help message and exit + -d, --dry-run Validate but do not run simulations +""" + + +def parse_cli_args(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--dry-run", + help="Validate but do not run simulations", + action="store_true", + default=False, + ) + + return parser.parse_args() diff --git a/pyphare/pyphare/core/phare_utilities.py b/pyphare/pyphare/core/phare_utilities.py index 78961c364..07e56d779 100644 --- a/pyphare/pyphare/core/phare_utilities.py +++ b/pyphare/pyphare/core/phare_utilities.py @@ -1,6 +1,11 @@ import math import numpy as np + +def debug_print(*args): + if __debug__: + print(*args) + def all_iterables(*args): """ return true if all arguments are either lists or tuples diff --git a/pyphare/pyphare/cpp/validate.py b/pyphare/pyphare/cpp/validate.py new file mode 100644 index 000000000..c28719f7c --- /dev/null +++ b/pyphare/pyphare/cpp/validate.py @@ -0,0 +1,87 @@ +import os +import sys +import json +import dataclasses +from pathlib import Path + +from pyphare.core import phare_utilities +import pyphare.cpp as cpp + +DOT_PHARE_DIR = Path(os.getcwd()) / ".phare" + + +def python_version_from(binary): + return phare_utilities.decode_bytes( + phare_utilities.run_cli_cmd(f"{binary} -V", check=True).stdout.strip() + ) + + +def check_build_config_is_runtime_compatible(strict=True): + try: + build_config: dict = cpp.build_config() + + if "PHARE_CONFIG_ERROR" in build_config: + return + + build_python_version = build_config["PYTHON_VERSION"] + current_python_version = python_version_from(sys.executable) + if build_python_version != current_python_version: + print("Inconsistency detected!") + print("Python during build and now are not the same!") + print("Build python version :", build_python_version) + print("Current python version:", current_python_version) + raise ValueError("Python version mismatch!") + + except RuntimeError as e: + print("Could not interrogate python versions") + print("Please see 'Having Issues' Section of ISSUES.TXT") + print("Actual error (consider adding to issue report):", e) + + except ValueError as e: + print(e) + if strict: + raise e + + +@dataclasses.dataclass +class RuntimeSettings: + python_version: str + python_binary: str + + +def try_system_binary(cli, log_to): + with open(log_to, "w") as f: + try: + proc = phare_utilities.run_cli_cmd(cli, check=True) + f.write(phare_utilities.decode_bytes(proc.stdout).strip()) + except Exception as e: + f.write(f"failed to run cli command {cli}\n") + f.write(f"error {e}") + + +def try_system_binaries(log_dir): + + try_system_binary("free -g", log_dir / "free_dash_g.txt") + try_system_binary("lscpu", log_dir / "lscpu.txt") + try_system_binary("hwloc-info", log_dir / "hwloc_info.txt") + + +def log_runtime_config(): + cpp_lib = cpp.cpp_lib() + + settings = RuntimeSettings( + python_binary=sys.executable, + python_version=python_version_from(sys.executable), + ) + + if cpp_lib.mpi_rank() == 0: + DOT_PHARE_DIR.mkdir(exist_ok=True, parents=True) + cpp_lib.mpi_barrier() + + RANK_DIR = DOT_PHARE_DIR / f"rank_{cpp_lib.mpi_rank()}" + RANK_DIR.mkdir(exist_ok=True) + + with open(RANK_DIR / "runtime_config.json", "w") as f: + json.dump(dataclasses.asdict(settings), f) + + try_system_binaries(RANK_DIR) diff --git a/pyphare/pyphare/pharein/maxwellian_fluid_model.py b/pyphare/pyphare/pharein/maxwellian_fluid_model.py index c588d1271..bfd7d2389 100644 --- a/pyphare/pyphare/pharein/maxwellian_fluid_model.py +++ b/pyphare/pyphare/pharein/maxwellian_fluid_model.py @@ -53,7 +53,8 @@ def __init__(self, bx = None, for population in self.populations: self.add_population(population, **kwargs[population]) - self.validate(global_vars.sim) + if not global_vars.sim.dry_run: + self.validate(global_vars.sim) global_vars.sim.set_model(self) @@ -142,7 +143,7 @@ def to_dict(self): #------------------------------------------------------------------------------ def validate(self, sim, atol=1e-15): - print(f"validating dim={sim.ndim}") + phare_utilities.debug_print(f"validating dim={sim.ndim}") if sim.ndim==1: self.validate1d(sim, atol) elif sim.ndim==2: @@ -213,11 +214,11 @@ def getCoord(L,R, idir): else: return (np.zeros_like(L),L), (np.zeros_like(R),R) - print("2d periodic validation") + phare_utilities.debug_print("2d periodic validation") for idir in np.arange(sim.ndim): - print("validating direction ...", idir) + phare_utilities.debug_print("validating direction ...", idir) if sim.boundary_types[idir] == "periodic": - print(f"direction {idir} is periodic?") + phare_utilities.debug_print(f"direction {idir} is periodic?") dual_left = (np.arange(-nbrDualGhosts, nbrDualGhosts)+0.5)*sim.dl[0] dual_right = dual_left + domain[0] primal_left = np.arange(-nbrPrimalGhosts, nbrPrimalGhosts)*sim.dl[0] @@ -247,12 +248,12 @@ def getCoord(L,R, idir): fL = f(*coordsL) fR = f(*coordsR) check = np.allclose(fL,fR, atol=atol, rtol=0) - print(f"checked {fn} : fL = {fL} and fR = {fR} and check = {check}") + phare_utilities.debug_print(f"checked {fn} : fL = {fL} and fR = {fR} and check = {check}") if not check: not_periodic += [(fn,idir)] is_periodic &=check if not is_periodic: - print("Warning: Simulation is periodic but some functions are not : ", not_periodic) + phare_utilities.debug_print("Warning: Simulation is periodic but some functions are not : ", not_periodic) if sim.strict: raise RuntimeError("Simulation is not periodic") diff --git a/pyphare/pyphare/pharein/simulation.py b/pyphare/pyphare/pharein/simulation.py index 396af7ca8..2c38f58c0 100644 --- a/pyphare/pyphare/pharein/simulation.py +++ b/pyphare/pyphare/pharein/simulation.py @@ -5,6 +5,12 @@ from . import global_vars from ..core import box as boxm from ..core.box import Box +from ..core import parse_cli_args + +CLI_ARGS = parse_cli_args() + + +# ------------------------------------------------------------------------------ def supported_dimensions(): @@ -498,7 +504,7 @@ def check_clustering(**kwargs): def checker(func): def wrapper(simulation_object, **kwargs): - accepted_keywords = ['domain_size', 'cells', 'dl', 'particle_pusher', 'final_time', + accepted_keywords = ['domain_size', 'cells', 'dl', 'particle_pusher', 'final_time', "dry_run", 'time_step', 'time_step_nbr', 'layout', 'interp_order', 'origin', 'boundary_types', 'refined_particle_nbr', 'path', 'nesting_buffer', 'diag_export_format', 'refinement_boxes', 'refinement', 'clustering', @@ -566,6 +572,8 @@ def wrapper(simulation_object, **kwargs): kwargs["hyper_resistivity"] = check_hyper_resistivity(**kwargs) + kwargs["dry_run"] = CLI_ARGS.dry_run + return func(simulation_object, **kwargs) return wrapper @@ -729,6 +737,7 @@ def __init__(self, **kwargs): validate_restart_options(self) + def final_time(self): return self.time_step * self.time_step_nbr diff --git a/pyphare/pyphare/simulator/simulator.py b/pyphare/pyphare/simulator/simulator.py index 949cf1643..a5306ce31 100644 --- a/pyphare/pyphare/simulator/simulator.py +++ b/pyphare/pyphare/simulator/simulator.py @@ -1,4 +1,6 @@ - +# +# +# import atexit import time as timem @@ -7,6 +9,7 @@ life_cycles = {} + @atexit.register def simulator_shutdown(): from ._simulator import obj @@ -57,6 +60,11 @@ def setup(self): from pyphare.cpp import cpp_lib import pyphare.pharein as ph startMPI() + + import pyphare.cpp.validate as validate_cpp + validate_cpp.log_runtime_config() + validate_cpp.check_build_config_is_runtime_compatible() + if self.log_to_file: self._log_to_file() ph.populateDict() @@ -79,6 +87,9 @@ def initialize(self): if self.cpp_hier is None: self.setup() + if self.simulation.dry_run: + return + self.cpp_sim.initialize() self._auto_dump() # first dump might be before first advance return self @@ -97,6 +108,8 @@ def _throw(self, e): def advance(self, dt = None): self._check_init() + if self.simulation.dry_run: + return if dt is None: dt = self.timeStep() @@ -119,6 +132,8 @@ def times(self): def run(self): from pyphare.cpp import cpp_lib self._check_init() + if self.simulation.dry_run: + return perf = [] end_time = self.cpp_sim.endTime() t = self.cpp_sim.currentTime() diff --git a/tools/config/config.py b/tools/config/config.py index 084806fdd..5f272ef8b 100644 --- a/tools/config/config.py +++ b/tools/config/config.py @@ -1,9 +1,8 @@ import os import json import subprocess -from pathlib import Path -from dataclasses import dataclass, field import dataclasses +from pathlib import Path FILE_DIR = Path(__file__).resolve().parent BUILD_DIR = FILE_DIR / "build" @@ -17,7 +16,7 @@ ) -@dataclass +@dataclasses.dataclass class SystemSettings: cmake_binary: str cmake_version: str @@ -73,7 +72,7 @@ class SystemSettings: def exec(cmd, on_error="error message"): try: - proc = subprocess.run(cmd.split(" "), check=False, capture_output=True) + proc = subprocess.run(cmd.split(" "), check=True, capture_output=True) return (proc.stdout + proc.stderr).decode().strip() except: return on_error @@ -187,7 +186,7 @@ def gen_system_file(): python_version=get_python_version(os.environ["PYTHON_EXECUTABLE"]), uname=exec("uname -a"), ) - with open(DOT_PHARE_DIR / ".build_config.json", "w") as f: + with open(DOT_PHARE_DIR / "build_config.json", "w") as f: json.dump(dataclasses.asdict(settings), f) with open(out_file, "w") as f: diff --git a/tools/report.py b/tools/report.py new file mode 100644 index 000000000..5bf4847d6 --- /dev/null +++ b/tools/report.py @@ -0,0 +1,61 @@ +import os +import sys +import shutil +import tempfile +import datetime +from pathlib import Path +from zipfile import ZipFile + +from tools.python3 import pushd + +SEARCH_PATHS = [os.getcwd()] + sys.path + +def find_phare_dirs(): + phare_dirs = [] + for path in SEARCH_PATHS: + print("path", path) + dot_phare_dir = Path(path) / ".phare" + if dot_phare_dir.exists(): + phare_dirs += [dot_phare_dir] + return phare_dirs + + +def main(): + phare_dirs = find_phare_dirs() + assert find_phare_dirs + cwd = Path(os.getcwd()) + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_dir = Path(tmpdirname) + for dir_idx, phare_dir in enumerate(phare_dirs): + with pushd(phare_dir.parent): + shutil.copytree(phare_dir.stem, tmp_dir / phare_dir.stem) + shutil.move( + tmp_dir / phare_dir.stem, f"{tmpdirname}/{phare_dir.stem}_{dir_idx}" + ) + + zip_name = "PHARE_REPORT.zip" + final_report_zip = cwd / zip_name + if final_report_zip.exists(): + """move existing to subdirectory with creation timestamp in name""" + reports_dir = cwd / ".phare_reports" + reports_dir.mkdir(exist_ok=True, parents=True) + timestamp = datetime.datetime.fromtimestamp( + final_report_zip.stat().st_ctime + ).isoformat() + shutil.move( + final_report_zip, + reports_dir / f"{final_report_zip.stem}.{timestamp}.zip", + ) + + tmp_report_zip = tmp_dir / zip_name + with pushd(tmpdirname): + """shutil.make_archive doesn't like multiple directory inputs""" + with ZipFile(tmp_report_zip, "w") as zip_object: + for phare_dir in Path(tmpdirname).glob(".phare_*"): + for item in phare_dir.glob("**/*"): + zip_object.write(item, arcname=item.relative_to(tmpdirname)) + shutil.move(tmp_report_zip, cwd) + + +if __name__ == "__main__": + main() diff --git a/tools/report.sh b/tools/report.sh new file mode 100755 index 000000000..fc474c6c7 --- /dev/null +++ b/tools/report.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# usage: +# ./tools/report.sh +# generates zip archive for logging issues + +set -eu +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT=$(cd "$SCRIPT_DIR/.." && pwd) +( + export PYTHONPATH="${PWD}:${PWD}/build:${PWD}/pyphare:${PYTHONPATH}" + python3 "$ROOT/tools/report.py" +)