Skip to content

Commit

Permalink
split base/full/slim
Browse files Browse the repository at this point in the history
  • Loading branch information
@david-leifker
david-leifker committed Aug 2, 2023
1 parent f2e2d72 commit 81df1d6
Showing 1 changed file with 91 additions and 35 deletions.
126 changes: 91 additions & 35 deletions .github/workflows/docker-unified.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ env:
DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup"
DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade"
DATAHUB_INGESTION_BASE_IMAGE: "acryldata/datahub-ingestion-base"
DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion"
DATAHUB_INGESTION_FULL_IMAGE: "acryldata/datahub-ingestion"
DATAHUB_INGESTION_SLIM_IMAGE: "acryldata/datahub-ingestion-slim"

jobs:
Expand Down Expand Up @@ -417,8 +417,8 @@ jobs:
file: ./docker/elasticsearch-setup/Dockerfile
platforms: linux/amd64,linux/arm64/v8

datahub_ingestion_build:
name: Build and Push DataHub Ingestion Docker Images
datahub_ingestion_base_build:
name: Build and Push DataHub Ingestion (Base) Docker Image
runs-on: ubuntu-latest
outputs:
tag: ${{ steps.tag.outputs.tag }}
Expand All @@ -434,10 +434,6 @@ jobs:
filters: |
datahub-ingestion-base:
- 'docker/datahub-ingestion-base/**'
datahub-ingestion-slim:
- 'docker/datahub-ingestion-slim/**'
datahub-ingestion:
- 'docker/datahub-ingestion/**'
- name: Build and push Base Image
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }}
uses: ./.github/actions/docker-custom-build-and-push
Expand All @@ -451,6 +447,29 @@ jobs:
context: .
file: ./docker/datahub-ingestion-base/Dockerfile
platforms: linux/amd64,linux/arm64/v8
- name: Compute DataHub Ingestion (Base) Tag
id: tag
run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT

datahub_ingestion_slim_build:
name: Build and Push DataHub Ingestion (Slim) Docker Images
runs-on: ubuntu-latest
outputs:
tag: ${{ steps.tag.outputs.tag }}
needs: [setup, datahub_ingestion_base_build]
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 800
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
datahub-ingestion-base:
- 'docker/datahub-ingestion-base/**'
datahub-ingestion-slim:
- 'docker/datahub-ingestion-slim/**'
- name: Build schemas
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true' }}
run: ./gradlew :metadata-events:mxe-schemas:build
Expand All @@ -469,12 +488,69 @@ jobs:
context: .
file: ./docker/datahub-ingestion-slim/Dockerfile
platforms: linux/amd64,linux/arm64/v8
- name: Compute (Slim) Tag
id: tag
run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT
datahub_ingestion_slim_scan:
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
name: "[Monitoring] Scan Datahub Ingestion Slim images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, datahub_ingestion_slim_build]
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image Slim Image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
- name: Run Trivy vulnerability scanner Slim Image
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"

datahub_ingestion_full_build:
name: Build and Push DataHub Ingestion (Full) Docker Images
runs-on: ubuntu-latest
outputs:
tag: ${{ steps.tag.outputs.tag }}
needs: [setup, datahub_ingestion_slim_build]
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 800
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
datahub-ingestion-base:
- 'docker/datahub-ingestion-base/**'
datahub-ingestion-slim:
- 'docker/datahub-ingestion-slim/**'
datahub-ingestion:
- 'docker/datahub-ingestion/**'
- name: Build and push Full Image
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }}
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_INGESTION_IMAGE }}
${{ env.DATAHUB_INGESTION_FULL_IMAGE }}
build-args: |
DOCKER_VERSION=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}
tags: ${{ needs.setup.outputs.tag }}
Expand All @@ -484,51 +560,31 @@ jobs:
context: .
file: ./docker/datahub-ingestion/Dockerfile
platforms: linux/amd64,linux/arm64/v8
- name: Compute Tag
- name: Compute Tag (Full)
id: tag
run: |
echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT
datahub_ingestion_scan:
run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT
datahub_ingestion_full_scan:
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, datahub_ingestion_build]
needs: [setup, datahub_ingestion_full_build]
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image Full Image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
image: ${{ env.DATAHUB_INGESTION_FULL_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
- name: Run Trivy vulnerability scanner Full Image
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Docker Prune
run: docker image prune "${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}"
- name: Download image Slim Image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
- name: Run Trivy vulnerability scanner Slim Image
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
image-ref: ${{ env.DATAHUB_INGESTION_FULL_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
Expand Down Expand Up @@ -558,7 +614,7 @@ jobs:
mae_consumer_build,
mce_consumer_build,
datahub_upgrade_build,
datahub_ingestion_build,
datahub_ingestion_slim_build,
]
steps:
- name: Disk Check
Expand Down

0 comments on commit 81df1d6

Please sign in to comment.