wip

src/anemoi/registry/workers.py

@@ -19,7 +19,7 @@
 
 
 class Worker:
-    def __init__(self, action, destination, timeout=None, wait=1, stop_if_finished=True, target_dir=".", request={}):
+    def __init__(self, action, destination, timeout=None, wait=60, stop_if_finished=True, target_dir=".", request={}):
         """Run a worker that will process tasks in the queue.
         timeout: Kill itself after `timeout` seconds.
         wait: When no task is found, wait `wait` seconds before checking again.
@@ -97,9 +97,6 @@ def get_source_path():
 
             path = locations[source]["path"]
 
-            if path.startswith("s3://"):
-                path = path + "/" if not path.endswith("/") else path
-
             return path
 
         source_path = get_source_path()
@@ -114,6 +111,9 @@ def get_source_path():
         LOG.info(f"Source path: {source_path}")
         LOG.info(f"Target path: {target_path}")
 
+        if source_path.startswith("s3://"):
+            source_path = source_path + "/" if not source_path.endswith("/") else source_path
+
         if target_path.startswith("s3://"):
             # untested
             download(source_path, target_path, resume=True)