Ensure an erroring ingestion produces a slack alert (#335)

* Ensure an erroring ingestion produces a slack alert * test threshold works
ministryofjustice · Nov 6, 2024 · 3a41867 · 3a41867
1 parent 38f2e25
commit 3a41867
Showing 1 changed file with 17 additions and 2 deletions.
diff --git a/.github/workflows/ingest-cadet-metadata.yml b/.github/workflows/ingest-cadet-metadata.yml
@@ -80,8 +80,12 @@ jobs:
           DATAHUB_GMS_URL: ${{ vars.DATAHUB_GMS_URL }}
           DATAHUB_TELEMETRY_ENABLED: false
         run: |
-          time poetry run datahub ingest -c ingestion/cadet.yaml --report-to output.json
-          FAILURE_COUNT=$(jq '.source.failures | length' output.json)
+          time poetry run datahub ingest -c ingestion/cadet.yaml --report-to output.json || true
+          if [ ! -f output.json ]; then
+            echo "Ingestion output file not found. Failing the job."
+            exit 1
+          fi
+          FAILURE_COUNT=$(jq '.sink.report.failures | length' output.json)
           echo "failure_count=$FAILURE_COUNT" >> $GITHUB_ENV
 
       - name: Notify on unexpected ingestion failures
@@ -94,3 +98,14 @@ jobs:
             }
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}
+
+      - name: Notify on ingestion not producing output.json or an unhandled failure
+        uses: slackapi/[email protected]
+        if: failure()
+        with:
+          payload: |
+            {
+                "text": ":warning: ALERT: DataHub CaDeT metadata ingestion produced no output on ${{inputs.ENVIRONMENT}} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}