From 3a418677acdab5cead94183145e682f4a8faebf0 Mon Sep 17 00:00:00 2001
From: Murdo <109604278+murdo-moj@users.noreply.github.com>
Date: Wed, 6 Nov 2024 16:49:32 +0000
Subject: [PATCH] Ensure an erroring ingestion produces a slack alert (#335)

* Ensure an erroring ingestion produces a slack alert

* test threshold works
---
 .github/workflows/ingest-cadet-metadata.yml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ingest-cadet-metadata.yml b/.github/workflows/ingest-cadet-metadata.yml
index 3631147..facee34 100644
--- a/.github/workflows/ingest-cadet-metadata.yml
+++ b/.github/workflows/ingest-cadet-metadata.yml
@@ -80,8 +80,12 @@ jobs:
           DATAHUB_GMS_URL: ${{ vars.DATAHUB_GMS_URL }}
           DATAHUB_TELEMETRY_ENABLED: false
         run: |
-          time poetry run datahub ingest -c ingestion/cadet.yaml --report-to output.json
-          FAILURE_COUNT=$(jq '.source.failures | length' output.json)
+          time poetry run datahub ingest -c ingestion/cadet.yaml --report-to output.json || true
+          if [ ! -f output.json ]; then
+            echo "Ingestion output file not found. Failing the job."
+            exit 1
+          fi
+          FAILURE_COUNT=$(jq '.sink.report.failures | length' output.json)
           echo "failure_count=$FAILURE_COUNT" >> $GITHUB_ENV
 
       - name: Notify on unexpected ingestion failures
@@ -94,3 +98,14 @@ jobs:
             }
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}
+
+      - name: Notify on ingestion not producing output.json or an unhandled failure
+        uses: slackapi/slack-github-action@v1.27.0
+        if: failure()
+        with:
+          payload: |
+            {
+                "text": ":warning: ALERT: DataHub CaDeT metadata ingestion produced no output on ${{inputs.ENVIRONMENT}} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}