diff --git a/mapchete/commands/_convert.py b/mapchete/commands/_convert.py index 07eb8e00..6903628e 100644 --- a/mapchete/commands/_convert.py +++ b/mapchete/commands/_convert.py @@ -25,6 +25,7 @@ from mapchete.io import MPath, fiona_open, get_best_zoom_level, rasterio_open, read_json from mapchete.io.vector import reproject_geometry from mapchete.tile import BufferedTilePyramid +from mapchete.types import MPathLike from mapchete.validate import validate_zooms logger = logging.getLogger(__name__) @@ -32,8 +33,8 @@ def convert( - tiledir: Union[str, dict, MPath], - output: Union[str, MPath], + tiledir: MPathLike, + output: MPathLike, zoom: Union[int, List[int]] = None, area: Union[BaseGeometry, str, dict] = None, area_crs: Union[CRS, str] = None, diff --git a/mapchete/commands/_execute.py b/mapchete/commands/_execute.py index 7d3938c8..a8931859 100644 --- a/mapchete/commands/_execute.py +++ b/mapchete/commands/_execute.py @@ -13,14 +13,15 @@ from mapchete.enums import Concurrency, ProcessingMode, Status from mapchete.errors import JobCancelledError from mapchete.executor import Executor +from mapchete.processing.profilers import pretty_bytes from mapchete.processing.types import TaskResult -from mapchete.types import Progress +from mapchete.types import MPathLike, Progress logger = logging.getLogger(__name__) def execute( - mapchete_config: Union[str, dict], + mapchete_config: Union[dict, MPathLike], zoom: Union[int, List[int]] = None, area: Union[BaseGeometry, str, dict] = None, area_crs: Union[CRS, str] = None, @@ -202,18 +203,16 @@ def execute( if print_task_details: msg = f"task {result.id}: {result.process_msg}" if result.profiling: # pragma: no cover - max_allocated = ( - result.profiling["memory"].max_allocated - / 1024 - / 1024 - ) + max_allocated = result.profiling["memory"].max_allocated head_requests = result.profiling["requests"].head_count get_requests = result.profiling["requests"].get_count requests = head_requests + get_requests - transfer = ( - result.profiling["requests"].get_bytes / 1024 / 1024 + transferred = result.profiling["requests"].get_bytes + msg += ( + f" (max memory usage: {pretty_bytes(max_allocated)}" ) - msg += f" (max memory usage: {max_allocated:.2f}MB, {requests} GET and HEAD requests, {transfer:.2f}MB transferred)" + msg += f", {requests} GET and HEAD requests" + msg += f", {pretty_bytes(transferred)} transferred)" all_observers.notify(message=msg) all_observers.notify( diff --git a/mapchete/processing/profilers/__init__.py b/mapchete/processing/profilers/__init__.py index a9bc68e4..c5f0cec7 100644 --- a/mapchete/processing/profilers/__init__.py +++ b/mapchete/processing/profilers/__init__.py @@ -10,3 +10,16 @@ Profiler(name="requests", decorator=measure_requests), Profiler(name="memory", decorator=measure_memory), ] + + +def pretty_bytes(count: float, round_value: int = 2) -> str: + """Return human readable bytes.""" + count = float(count) + + for measurement in ["bytes", "KiB", "MiB", "GiB", "TiB"]: + out = f"{round(count, round_value)} {measurement}" + if count < 1024.0: + break + count /= 1024.0 + + return out