From 19e0a3afdee321ad25040cb66910184af8778626 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Thu, 4 Jul 2024 15:35:14 +0200 Subject: [PATCH] wip --- src/anemoi/registry/workers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/anemoi/registry/workers.py b/src/anemoi/registry/workers.py index 4552257..bb01fdd 100644 --- a/src/anemoi/registry/workers.py +++ b/src/anemoi/registry/workers.py @@ -19,7 +19,7 @@ class Worker: - def __init__(self, action, destination, timeout=None, wait=1, stop_if_finished=True, target_dir=".", request={}): + def __init__(self, action, destination, timeout=None, wait=60, stop_if_finished=True, target_dir=".", request={}): """Run a worker that will process tasks in the queue. timeout: Kill itself after `timeout` seconds. wait: When no task is found, wait `wait` seconds before checking again. @@ -97,9 +97,6 @@ def get_source_path(): path = locations[source]["path"] - if path.startswith("s3://"): - path = path + "/" if not path.endswith("/") else path - return path source_path = get_source_path() @@ -114,6 +111,9 @@ def get_source_path(): LOG.info(f"Source path: {source_path}") LOG.info(f"Target path: {target_path}") + if source_path.startswith("s3://"): + source_path = source_path + "/" if not source_path.endswith("/") else source_path + if target_path.startswith("s3://"): # untested download(source_path, target_path, resume=True)