fix yaml path and add a gh action

.github/workflows/build_prod_test.yml

@@ -0,0 +1,23 @@
+name: Test Prod Build
+
+on:
+    workflow_dispatch: # Allows manual triggering of the workflow
+    push:
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    test-build-prod:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout Code
+              uses: actions/checkout@v4
+
+            - name: Set up Docker
+              uses: docker/setup-buildx-action@v2
+
+            - name: Launch Docker Compose
+              run: python3 main.py prod --detach

action.yml

@@ -20,7 +20,7 @@ runs:
     - name: Set up Docker
       uses: docker/setup-buildx-action@v2
 
-    - name: Launch Docker Stack
+    - name: Launch Docker Compose
       run: python3 main.py dev --detach
       shell: bash
 
@@ -39,19 +39,7 @@ runs:
           exit 1
         fi
 
-    - name: Wait for MinIO
-      shell: bash
-      run: |
-        for i in {1..10}; do
-          if curl --max-time 10 -fs http://127.0.0.1:9001/minio/health/live; then
-            echo "MinIO is up!"
-            exit 0
-          fi
-          echo "Waiting for MinIO to be ready... Attempt $i"
-          sleep 10
-        done
-        echo "Failed to connect to MinIO after 10 attempts."
-        exit 1
+
 
     - name: Run tests that don't require secrets
       shell: bash

main.py

@@ -20,8 +20,9 @@ def run_subprocess(command: str, returnStdoutAsValue: bool = False, wait: bool =
         stdout=subprocess.PIPE if returnStdoutAsValue else sys.stdout,
         stderr=sys.stderr,
     )
-    stdout, _ = process.communicate()
+    stdout, stderr = process.communicate()
     if process.returncode != 0:
+        print(stderr.decode("utf-8"))
         sys.exit(process.returncode)
     return stdout.decode("utf-8") if returnStdoutAsValue else None
 

userCode/lib/env.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 from dagster import get_dagster_logger
 
@@ -96,7 +97,9 @@ def strict_env(key: str):
 LAKEFS_ACCESS_KEY_ID = strict_env("LAKEFS_ACCESS_KEY_ID")
 LAKEFS_SECRET_ACCESS_KEY = strict_env("LAKEFS_SECRET_ACCESS_KEY")
 
-DAGSTER_YAML_CONFIG = os.path.join(
-    os.path.dirname(__file__), "..", "..", "dagster.yaml"
-)
+_DAGSTER_HOME = Path(
+    os.environ.get("DAGSTER_HOME", Path(__file__) / "../../..")
+).resolve()
+DAGSTER_YAML_CONFIG = _DAGSTER_HOME / "dagster.yaml"
+
 assert os.path.exists(DAGSTER_YAML_CONFIG), "the dagster.yaml file does not exist"

userCode/lib/utils.py

@@ -20,6 +20,7 @@
     sources_partitions_def,
 )
 from .classes import S3
+from .types import cli_modes
 from .env import (
     GLEANERIO_DATAGRAPH_ENDPOINT,
     GLEANERIO_PROVGRAPH_ENDPOINT,
@@ -51,7 +52,7 @@ def run_scheduler_docker_image(
     source: str,  # which organization we are crawling
     image_name: str,  # the name of the docker image to pull and validate
     args: list[str],  # the list of arguments to pass to the gleaner/nabu command
-    action_name: str,  # the name of the action to run inside gleaner/nabu
+    action_name: cli_modes,  # the name of the action to run inside gleaner/nabu
     volumeMapping: Optional[list[str]] = None,
 ):
     """Run a docker image inside the dagster docker runtime"""

userCode/main.py

@@ -195,9 +195,9 @@ def gleaner_config(context: AssetExecutionContext):
     sources = []
     names: set[str] = set()
 
-    assert (
-        len(Lines) > 0
-    ), f"No sitemaps found in sitemap index {REMOTE_GLEANER_SITEMAP}"
+    assert len(Lines) > 0, (
+        f"No sitemaps found in sitemap index {REMOTE_GLEANER_SITEMAP}"
+    )
 
     for line in Lines:
         basename = REMOTE_GLEANER_SITEMAP.removesuffix(".xml")
@@ -459,7 +459,7 @@ def nabu_prov_object(context):
         GLEANERIO_PROVGRAPH_ENDPOINT,
     ]
     run_scheduler_docker_image(
-        source,
+        "source",
         NABU_IMAGE,
         ARGS,
         "prov-object",