Merge pull request #126 from neojunjie/revert-122-refactor_ingestion

Revert "feat(datahub-ingestion): refactor datahub ingestion slim images (datahub-project#8515)"
aqden · Oct 2, 2023 · e1330f7 · e1330f7
2 parents a9c0f41 + 44922c5
commit e1330f7
Show file tree

Hide file tree

Showing 34 changed files with 406 additions and 637 deletions.
diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml
@@ -30,9 +30,6 @@ inputs:
     # e.g. latest,head,sha12345
     description: "List of tags to use for the Docker image"
     required: true
-  target:
-    description: "Sets the target stage to build"
-    required: false
 outputs:
   image_tag:
     description: "Docker image tags"
@@ -65,7 +62,6 @@ runs:
         platforms: linux/amd64
         build-args: ${{ inputs.build-args }}
         tags: ${{ steps.docker_meta.outputs.tags }}
-        target: ${{ inputs.target }}
         load: true
         push: false
     - name: Upload image locally for testing (if not publishing)
@@ -96,7 +92,6 @@ runs:
         platforms: ${{ inputs.platforms }}
         build-args: ${{ inputs.build-args }}
         tags: ${{ steps.docker_meta.outputs.tags }}
-        target: ${{ inputs.target }}
         push: true
 
     # TODO add code for vuln scanning?
diff --git a/.github/workflows/docker-ingestion-base.yml b/.github/workflows/docker-ingestion-base.yml
@@ -0,0 +1,45 @@
+name: ingestion base
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/docker-ingestion-base.yml"
+      - "docker/datahub-ingestion-base/**"
+      - "gradle*"
+  pull_request:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/docker-ingestion-base.yml"
+      - "docker/datahub-ingestion-base/**"
+      - "gradle*"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-base:
+    name: Build and Push Docker Image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Build and Push image
+        uses: ./.github/actions/docker-custom-build-and-push
+        with:
+          images: |
+            heruko/datahub-ingestion-base
+          tags: latest
+          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
+          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
+          publish: ${{ github.ref == 'refs/heads/master' }}
+          context: .
+          file: ./docker/datahub-ingestion-base/Dockerfile
+          platforms: linux/amd64,linux/arm64/v8
diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml
@@ -0,0 +1,118 @@
+name: datahub-ingestion docker
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+      - "docker/datahub-ingestion/**"
+      - "docker/datahub-ingestion-slim/**"
+      - ".github/workflows/docker-ingestion.yml"
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+      publish: ${{ steps.publish.outputs.publish }}
+      python_release_version: ${{ steps.python_release_version.outputs.release_version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Compute Tag
+        id: tag
+        run: |
+          source .github/scripts/docker_helpers.sh
+          echo "tag=$(get_tag)" >> $GITHUB_OUTPUT
+      - name: Compute Python Release Version
+        id: python_release_version
+        run: |
+          source .github/scripts/docker_helpers.sh
+          echo "release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
+      - name: Check whether publishing enabled
+        id: publish
+        env:
+          ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
+          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
+  push_to_registries:
+    name: Build and Push Docker Image to Docker Hub
+    runs-on: ubuntu-latest
+    needs: setup
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Build and push
+        uses: ./.github/actions/docker-custom-build-and-push
+        with:
+          images: |
+            heruko/datahub-ingestion
+          tags: ${{ needs.setup.outputs.tag }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          publish: ${{ needs.setup.outputs.publish == 'true' }}
+          context: .
+          file: ./docker/datahub-ingestion/Dockerfile
+          platforms: linux/amd64,linux/arm64/v8
+          build-args: |
+            RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
+      - name: Build and Push image (slim)
+        uses: ./.github/actions/docker-custom-build-and-push
+        with:
+          images: |
+            heruko/datahub-ingestion-slim
+          tags: ${{ needs.setup.outputs.tag }}
+          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
+          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
+          publish: ${{ needs.setup.outputs.publish == 'true' }}
+          context: .
+          file: ./docker/datahub-ingestion-slim/Dockerfile
+          platforms: linux/amd64,linux/arm64/v8
+  ingestion-slim_scan:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+    name: "[Monitoring] Scan datahub-ingestion-slim images for vulnerabilities"
+    if: ${{ github.ref == 'refs/heads/master' }}
+    runs-on: ubuntu-latest
+    needs: [push_to_registries]
+    steps:
+      - name: Checkout # adding checkout step just to make trivy upload happy
+        uses: actions/checkout@v3
+      - name: Download image
+        uses: ishworkh/docker-image-artifact-download@v1
+        with:
+          image: heruko/datahub-ingestion-slim:latest
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/[email protected]
+        env:
+          TRIVY_OFFLINE_SCAN: true
+        with:
+          image-ref: heruko/datahub-ingestion-slim:latest
+          format: "template"
+          template: "@/contrib/sarif.tpl"
+          output: "trivy-results.sarif"
+          severity: "CRITICAL,HIGH"
+          ignore-unfixed: true
+          vuln-type: "os,library"
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"