feat: ETL testing in PRs, separate deployment triggers from core apps (…

…#1424)
bcgov · Jul 22, 2024 · 6133dd8 · 6133dd8
1 parent 8d409a2
commit 6133dd8
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 57 deletions.
diff --git a/.github/workflows/.deploy.yml b/.github/workflows/.deploy.yml
@@ -22,47 +22,54 @@ on:
         default: ${{ github.event.number }}
         required: false
         type: string
-      triggers:
-        description: Deployment trigger; omit = build; e.g. ('backend/' 'frontend/')
-        required: false
-        type: string
-      ### Usually a bad idea / not recommended
-      timeout-minutes:
-        description: 'Timeout minutes'
-        default: 10
-        required: false
-        type: number
 
     outputs:
-      triggered:
-        description: 'Has a deployment has been triggered?'
-        value: ${{ jobs.deploy.outputs.triggered }}
+      run_tests:
+        description: Run Cypress tests if the core apps have changed (excludes sync)
+        value: ${{ jobs.init.outputs.deploy_core }}
 
 jobs:
   init:
     name: Deploy (init)
     environment: ${{ inputs.environment }}
     outputs:
-      mod-tag: ${{ steps.mod-tag.outputs.mod-tag }}
-      triggered: ${{ steps.triggers.outputs.triggered }}
+      fam-modded-zone: ${{ steps.fam-modded-zone.outputs.fam-modded-zone }}
+      deploy_core: ${{ steps.triggers.outputs.core }}
+      deploy_sync: ${{ steps.triggers.outputs.sync }}
     runs-on: ubuntu-22.04
     steps:
       # Check triggers (omitted or matched)
-      - uses: bcgov-nr/[email protected]
-        id: triggers
+      - name: Check core triggers
+        uses: bcgov-nr/[email protected]
+        id: check_core
         with:
-          triggers: ${{ inputs.triggers }}
+          triggers: ('backend/' 'common/' 'database/' 'frontend/' 'oracle-api/')
 
-      - if: steps.triggers.outputs.triggered == 'false'
-        run: echo "No deployment required/triggered!"
+      - name: Check sync triggers
+        uses: bcgov-nr/[email protected]
+        id: check_sync
+        with:
+          triggers: ('common/' 'sync/')
 
-      - name: Get PR Number Mod 50
-        if: github.event_name == 'pull_request' && steps.triggers.outputs.triggered == 'true'
-        id: mod-tag
-        run: echo "mod-tag=$(( ${{ inputs.target }} % 50 ))" >> $GITHUB_OUTPUT
+      # Simplify triggers
+      - name: Simplify triggers
+        id: triggers
+        run: |
+          echo "core=${{ github.event_name != 'pull_request' || steps.check_core.outputs.triggered == 'true' }}" >> $GITHUB_OUTPUT
+          echo "sync=${{ github.event_name != 'pull_request' || steps.check_sync.outputs.triggered == 'true' }}" >> $GITHUB_OUTPUT
+
+      - name: FAM routing
+        id: fam-modded-zone
+        if: steps.triggers.outputs.core == 'true'
+        run: |
+          if [ ${{ github.event_name }} == 'pull_request' ]; then
+            echo "fam-modded-zone=$(( ${{ inputs.target }} % 50 ))" >> $GITHUB_OUTPUT
+          else
+            echo "fam-modded-zone=${{ inputs.target }}" >> $GITHUB_OUTPUT
+          fi
 
       - name: OpenShift Init
-        if: github.event_name != 'pull_request' || steps.triggers.outputs.triggered == 'true'
+        if: steps.triggers.outputs.core == 'true' || steps.triggers.outputs.sync == 'true'
         uses: bcgov-nr/[email protected]
         with:
           oc_namespace: ${{ vars.OC_NAMESPACE }}
@@ -80,14 +87,8 @@ jobs:
             -p ORACLE_SYNC_PASSWORD='${{ secrets.ORACLE_SYNC_PASSWORD }}'
             -p ORACLE_CERT_SECRET='${{ secrets.ORACLE_CERT_SECRET }}'
 
-  database:
-    name: Deploy (database)
-    environment: ${{ inputs.environment }}
-    if: github.event_name != 'pull_request' || needs.init.outputs.triggered == 'true'
-    needs: [init]
-    runs-on: ubuntu-22.04
-    steps:
       - name: Database
+        if: steps.triggers.outputs.core == 'true' || steps.triggers.outputs.sync == 'true'
         uses: bcgov-nr/[email protected]
         with:
           oc_namespace: ${{ vars.OC_NAMESPACE }}
@@ -103,16 +104,14 @@ jobs:
 
   deploy:
     name: Deploy
+    if: needs.init.outputs.deploy_core == 'true'
     environment: ${{ inputs.environment }}
-    if: github.event_name != 'pull_request' || needs.init.outputs.triggered == 'true'
-    needs: [database, init]
-    outputs:
-      triggered: ${{ steps.trigger.outputs.triggered }}
+    needs: [init]
     runs-on: ubuntu-22.04
-    timeout-minutes: ${{ inputs.timeout-minutes }}
+    timeout-minutes: 10
     strategy:
       matrix:
-        name: [backend, frontend, oracle-api, sync]
+        name: [backend, frontend, oracle-api]
         include:
           - name: backend
             file: backend/openshift.deploy.yml
@@ -124,7 +123,7 @@ jobs:
             file: frontend/openshift.deploy.yml
             overwrite: true
             parameters:
-              -p FAM_MODDED_ZONE=${{ github.event_name == 'pull_request' && needs.init.outputs.mod-tag || inputs.target }}
+              -p FAM_MODDED_ZONE=${{ needs.init.outputs.fam-modded-zone }}
               -p VITE_SPAR_BUILD_VERSION=snapshot-${{ inputs.target || github.event.number }}
               -p VITE_USER_POOLS_ID=${{ vars.VITE_USER_POOLS_ID }}
               -p VITE_USER_POOLS_WEB_CLIENT_ID=${{ vars.VITE_USER_POOLS_WEB_CLIENT_ID }}
@@ -134,9 +133,6 @@ jobs:
             parameters:
               -p AWS_COGNITO_ISSUER_URI=https://cognito-idp.ca-central-1.amazonaws.com/${{ vars.VITE_USER_POOLS_ID }}
             verification_path: "actuator/health"
-          - name: sync
-            file: sync/openshift.deploy.yml
-            overwrite: true
 
     steps:
       - uses: bcgov-nr/[email protected]
@@ -146,18 +142,70 @@ jobs:
           oc_namespace: ${{ vars.OC_NAMESPACE }}
           oc_server: ${{ vars.OC_SERVER }}
           oc_token: ${{ secrets.OC_TOKEN }}
+          oc_version: ${{ matrix.oc_version }}
           overwrite: ${{ matrix.overwrite }}
-          penetration_test: false
           parameters:
             -p TAG=${{ inputs.tag }}
             -p ZONE=${{ inputs.target }}
             ${{ github.event_name == 'pull_request' && '-p MIN_REPLICAS=1' || '' }}
             ${{ github.event_name == 'pull_request' && '-p MAX_REPLICAS=2' || '' }}
             ${{ matrix.parameters }}
-          post_rollout: ${{ matrix.post_rollout }}
-          triggers: ${{ inputs.triggers }}
           verification_path: ${{ matrix.verification_path }}
-          verification_retry_attempts: "5"
-          verification_retry_seconds: "20"
-      - id: trigger
-        run: echo "triggered=${{ steps.deploys.outputs.triggered }}" >> $GITHUB_OUTPUT
+          verification_retry_attempts: 5
+          verification_retry_seconds: 20
+
+  # ETL testing will only run on Pull Requests if the sync/ directory is modified
+  sync:
+    name: Deploy (sync)
+    if: needs.init.outputs.deploy_sync == 'true'
+    needs: [init]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy (sync)
+        uses: bcgov-nr/[email protected]
+        with:
+          file: sync/openshift.deploy.yml
+          oc_namespace: ${{ vars.OC_NAMESPACE }}
+          oc_server: ${{ vars.OC_SERVER }}
+          oc_token: ${{ secrets.OC_TOKEN }}
+          oc_version: "4.13"
+          overwrite: true
+          parameters:
+            -p TAG=${{ inputs.tag }}
+            -p ZONE=${{ inputs.target }}
+            -p TEST_MODE=true
+
+      - name: Override OpenShift version
+        if: github.event_name == 'pull_request'
+        env:
+          OC: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable-4.13/openshift-client-linux.tar.gz
+        run: |
+          # Download and extract with retry, continuing on error
+          (wget ${{ env.OC }} -qcO - | tar -xzvf - oc)|| !! || true
+          oc version
+        working-directory: /usr/local/bin/
+
+      - name: Run sync ETL
+        if: github.event_name == 'pull_request'
+        run: |
+          # Run and verify job
+
+          # Login
+          oc login --token=${{ secrets.oc_token }} --server=${{ vars.oc_server }}
+          oc project ${{ secrets.oc_namespace }} #Safeguard!
+
+          # Exit on errors or unset variables
+          set -eu
+
+          # Create job
+          CRONJOB=nr-spar-${{ inputs.target }}-sync
+          RUN_JOB=${CRONJOB}--$(date +"%Y-%m-%d--%H-%M-%S")
+          oc create job ${RUN_JOB} --from=cronjob/${CRONJOB}
+
+          # Follow
+          oc wait --for=condition=ready pod --selector=job-name=${RUN_JOB} --timeout=1m
+          oc logs -l job-name=${RUN_JOB} --tail=50 --follow
+
+          # Verify successful completion
+          oc wait --for jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=${RUN_JOB} --timeout=1m
+          echo "Job successful!"
diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml
@@ -33,16 +33,13 @@ jobs:
     needs: [builds]
     secrets: inherit
     uses: ./.github/workflows/.deploy.yml
-    with:
-      triggers: ('backend/' 'common/' 'database/' 'frontend/' 'oracle-api/' 'sync/')
 
   tests:
     name: Tests
+    if: needs.deploys.outputs.run_tests == 'true'
     needs: [deploys]
     secrets: inherit
     uses: ./.github/workflows/.tests.yml
-    with:
-      triggers: ('backend/' 'common/' 'database/' 'frontend/' 'oracle-api/')
 
   results:
     name: PR Results

diff --git a/sync/Dockerfile b/sync/Dockerfile
@@ -13,7 +13,7 @@ ENV PYTHONDONTWRITEBYTECODE="1"
 # Copy files and install requirements
 WORKDIR /app
 COPY . ./
-RUN pip install -r ./requirements.txt
+RUN pip install -r ./requirements.txt --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host files.pythonhosted.org
 
 # Start the app
 USER nonroot

diff --git a/sync/openshift.deploy.yml b/sync/openshift.deploy.yml
@@ -44,9 +44,6 @@ parameters:
     description: Source container registry
     value: ghcr.io
 
-  ### Dummy values for workflow convenience
-  - name: MIN_REPLICAS
-  - name: MAX_REPLICAS
 objects:
   - kind: CronJob
     apiVersion: "batch/v1"