fix config dir

action.yml

@@ -60,4 +60,4 @@ runs:
         python -m venv .venv
         source .venv/bin/activate
         pip install -r requirements.txt
-        pytest
+        pytest -x

userCode/lib/classes.py

@@ -54,7 +54,8 @@ def load(self, data: Any, remote_path: str):
 
     def read(self, remote_path: str):
         logger = get_dagster_logger()
-        logger.info(f"S3 FULL SERVER : {self.endpoint}")
+        logger.info(f"S3 endpoint that dagster will connect to: {self.endpoint}")
+        logger.info(f"S3 Address that gleaner will use: {GLEANER_MINIO_ADDRESS}")
         logger.info(f"S3 BUCKET : {GLEANER_MINIO_BUCKET}")
         logger.debug(f"S3 object path : {remote_path}")
         response: BaseHTTPResponse = self.client.get_object(

userCode/main.py

@@ -193,9 +193,9 @@ def gleaner_config(context: AssetExecutionContext):
     sources = []
     names: set[str] = set()
 
-    assert (
-        len(Lines) > 0
-    ), f"No sitemaps found in sitemap index {REMOTE_GLEANER_SITEMAP}"
+    assert len(Lines) > 0, (
+        f"No sitemaps found in sitemap index {REMOTE_GLEANER_SITEMAP}"
+    )
 
     for line in Lines:
         basename = REMOTE_GLEANER_SITEMAP.removesuffix(".xml")
@@ -328,7 +328,7 @@ def can_contact_headless():
 def gleaner(context: OpExecutionContext):
     """Get the jsonld for each site in the gleaner config"""
     source = context.partition_key
-    ARGS = ["--cfg", "gleanerconfig.yaml", "--source", source, "--rude"]
+    ARGS = ["--cfg", "gleanerconfig.yaml", "--source", source, "--rude", "--setup"]
 
     returned_value = run_scheduler_docker_image(
         context,
@@ -358,7 +358,7 @@ def nabu_release(context: OpExecutionContext):
         NABU_IMAGE,
         ARGS,
         "release",
-        volumeMapping=["/tmp/nabuconfig.yaml:/app/nabuconfig.yaml"],
+        volumeMapping=["/tmp/nabuconfig.yaml:/nabuconfig.yaml"],
     )
 
 
@@ -380,7 +380,7 @@ def nabu_object(context: OpExecutionContext):
         NABU_IMAGE,
         ARGS,
         "object",
-        volumeMapping=["/tmp/nabuconfig.yaml:/app/nabuconfig.yaml"],
+        volumeMapping=["/tmp/nabuconfig.yaml:/nabuconfig.yaml"],
     )
 
 
@@ -403,7 +403,7 @@ def nabu_prune(context: OpExecutionContext):
         NABU_IMAGE,
         ARGS,
         "prune",
-        volumeMapping=["/tmp/nabuconfig.yaml:/app/nabuconfig.yaml"],
+        volumeMapping=["/tmp/nabuconfig.yaml:/nabuconfig.yaml"],
     )
 
 
@@ -425,7 +425,7 @@ def nabu_prov_release(context):
         NABU_IMAGE,
         ARGS,
         "prov-release",
-        volumeMapping=["/tmp/nabuconfig.yaml:/app/nabuconfig.yaml"],
+        volumeMapping=["/tmp/nabuconfig.yaml:/nabuconfig.yaml"],
     )
 
 
@@ -446,7 +446,7 @@ def nabu_prov_clear(context: OpExecutionContext):
         NABU_IMAGE,
         ARGS,
         "prov-clear",
-        volumeMapping=["/tmp/nabuconfig.yaml:/app/nabuconfig.yaml"],
+        volumeMapping=["/tmp/nabuconfig.yaml:/nabuconfig.yaml"],
     )
 
 
@@ -468,7 +468,7 @@ def nabu_prov_object(context):
         NABU_IMAGE,
         ARGS,
         "prov-object",
-        volumeMapping=["/tmp/nabuconfig.yaml:/app/nabuconfig.yaml"],
+        volumeMapping=["/tmp/nabuconfig.yaml:/nabuconfig.yaml"],
     )
 
 
@@ -492,7 +492,7 @@ def nabu_orgs_release(context: OpExecutionContext):
         NABU_IMAGE,
         ARGS,
         "orgs-release",
-        volumeMapping=["/tmp/nabuconfig.yaml:/app/nabuconfig.yaml"],
+        volumeMapping=["/tmp/nabuconfig.yaml:/nabuconfig.yaml"],
     )
 
 
@@ -515,7 +515,7 @@ def nabu_orgs(context: OpExecutionContext):
         NABU_IMAGE,
         ARGS,
         "orgs",
-        volumeMapping=["/tmp/nabuconfig.yaml:/app/nabuconfig.yaml"],
+        volumeMapping=["/tmp/nabuconfig.yaml:/nabuconfig.yaml"],
     )