From 519d63b44156559fa87d85cf613f0e86b9158750 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Thu, 9 Nov 2023 14:47:36 +0100 Subject: [PATCH] better progress messages --- mapchete/commands/_execute.py | 12 +++++++++++- mapchete/processing/profilers/requests.py | 2 +- mapchete/processing/types.py | 3 ++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mapchete/commands/_execute.py b/mapchete/commands/_execute.py index 29002b78..4662383f 100644 --- a/mapchete/commands/_execute.py +++ b/mapchete/commands/_execute.py @@ -221,7 +221,17 @@ def _process_everything( for future in mp.compute(**kwargs): result = TaskResult.from_future(future) if print_task_details: - msg_callback(str(result)) + msg = f"task {result.id}: {result.process_msg}" + if result.profiling: + max_allocated = ( + result.profiling["memory"].max_allocated / 1024 / 1024 + ) + head_requests = result.profiling["requests"].head_count + get_requests = result.profiling["requests"].get_count + requests = head_requests + get_requests + transfer = result.profiling["requests"].get_bytes / 1024 / 1024 + msg += f" (max memory usage: {max_allocated:.2f}MB, {requests} GET and HEAD requests, {transfer:.2f}MB transferred)" + msg_callback(msg) yield result # explicitly exit the mp object on success mp.__exit__(None, None, None) diff --git a/mapchete/processing/profilers/requests.py b/mapchete/processing/profilers/requests.py index 313c4c1b..5e10ff62 100644 --- a/mapchete/processing/profilers/requests.py +++ b/mapchete/processing/profilers/requests.py @@ -26,7 +26,7 @@ def wrapped_f(*args, **kwargs) -> Union[Any, Tuple[Any, MeasuredRequests]]: "please install tilebench if you want to use this feature." ) - @profile(add_to_return=True) + @profile(add_to_return=True, quiet=True) def _decorated(func, fargs, fkwargs) -> Any: return func(*fargs, **fkwargs) diff --git a/mapchete/processing/types.py b/mapchete/processing/types.py index 6b970610..58b7a864 100644 --- a/mapchete/processing/types.py +++ b/mapchete/processing/types.py @@ -63,8 +63,9 @@ def from_future(future: MFuture) -> TaskResult: profiling=process_info.profiling or future.profiling, ) elif isinstance(process_info, TileProcessInfo): + tile = process_info.tile return TaskResult( - id=process_info.tile, + id=f"tile-{tile.zoom}-{tile.row}-{tile.col}", processed=process_info.processed, process_msg=process_info.process_msg, profiling=process_info.profiling or future.profiling,