Skip to content

Commit

Permalink
split base/full/slim
Browse files Browse the repository at this point in the history
  • Loading branch information
@david-leifker
david-leifker committed Aug 3, 2023
1 parent f2e2d72 commit 2dc6b78
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 46 deletions.
144 changes: 99 additions & 45 deletions .github/workflows/docker-unified.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ env:
DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup"
DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade"
DATAHUB_INGESTION_BASE_IMAGE: "acryldata/datahub-ingestion-base"
DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion"
DATAHUB_INGESTION_FULL_IMAGE: "acryldata/datahub-ingestion"
DATAHUB_INGESTION_SLIM_IMAGE: "acryldata/datahub-ingestion-slim"

jobs:
Expand Down Expand Up @@ -417,8 +417,8 @@ jobs:
file: ./docker/elasticsearch-setup/Dockerfile
platforms: linux/amd64,linux/arm64/v8

datahub_ingestion_build:
name: Build and Push DataHub Ingestion Docker Images
datahub_ingestion_base_build:
name: Build and Push DataHub Ingestion (Base) Docker Image
runs-on: ubuntu-latest
outputs:
tag: ${{ steps.tag.outputs.tag }}
Expand All @@ -434,10 +434,6 @@ jobs:
filters: |
datahub-ingestion-base:
- 'docker/datahub-ingestion-base/**'
datahub-ingestion-slim:
- 'docker/datahub-ingestion-slim/**'
datahub-ingestion:
- 'docker/datahub-ingestion/**'
- name: Build and push Base Image
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }}
uses: ./.github/actions/docker-custom-build-and-push
Expand All @@ -451,6 +447,29 @@ jobs:
context: .
file: ./docker/datahub-ingestion-base/Dockerfile
platforms: linux/amd64,linux/arm64/v8
- name: Compute DataHub Ingestion (Base) Tag
id: tag
run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT

datahub_ingestion_slim_build:
name: Build and Push DataHub Ingestion (Slim) Docker Images
runs-on: ubuntu-latest
outputs:
tag: ${{ steps.tag.outputs.tag }}
needs: [setup, datahub_ingestion_base_build]
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 800
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
datahub-ingestion-base:
- 'docker/datahub-ingestion-base/**'
datahub-ingestion-slim:
- 'docker/datahub-ingestion-slim/**'
- name: Build schemas
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true' }}
run: ./gradlew :metadata-events:mxe-schemas:build
Expand All @@ -469,12 +488,69 @@ jobs:
context: .
file: ./docker/datahub-ingestion-slim/Dockerfile
platforms: linux/amd64,linux/arm64/v8
- name: Compute (Slim) Tag
id: tag
run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT
datahub_ingestion_slim_scan:
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
name: "[Monitoring] Scan Datahub Ingestion Slim images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, datahub_ingestion_slim_build]
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image Slim Image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
- name: Run Trivy vulnerability scanner Slim Image
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"

datahub_ingestion_full_build:
name: Build and Push DataHub Ingestion (Full) Docker Images
runs-on: ubuntu-latest
outputs:
tag: ${{ steps.tag.outputs.tag }}
needs: [setup, datahub_ingestion_slim_build]
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 800
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
datahub-ingestion-base:
- 'docker/datahub-ingestion-base/**'
datahub-ingestion-slim:
- 'docker/datahub-ingestion-slim/**'
datahub-ingestion:
- 'docker/datahub-ingestion/**'
- name: Build and push Full Image
if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }}
uses: ./.github/actions/docker-custom-build-and-push
with:
images: |
${{ env.DATAHUB_INGESTION_IMAGE }}
${{ env.DATAHUB_INGESTION_FULL_IMAGE }}
build-args: |
DOCKER_VERSION=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}
tags: ${{ needs.setup.outputs.tag }}
Expand All @@ -484,51 +560,31 @@ jobs:
context: .
file: ./docker/datahub-ingestion/Dockerfile
platforms: linux/amd64,linux/arm64/v8
- name: Compute Tag
- name: Compute Tag (Full)
id: tag
run: |
echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT
datahub_ingestion_scan:
run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT
datahub_ingestion_full_scan:
permissions:
contents: read # for actions/checkout to fetch code
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, datahub_ingestion_build]
needs: [setup, datahub_ingestion_full_build]
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: actions/checkout@v3
- name: Download image Full Image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
image: ${{ env.DATAHUB_INGESTION_FULL_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
- name: Run Trivy vulnerability scanner Full Image
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "CRITICAL,HIGH"
ignore-unfixed: true
vuln-type: "os,library"
- name: Docker Prune
run: docker image prune "${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}"
- name: Download image Slim Image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
- name: Run Trivy vulnerability scanner Slim Image
uses: aquasecurity/[email protected]
env:
TRIVY_OFFLINE_SCAN: true
with:
image-ref: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}
image-ref: ${{ env.DATAHUB_INGESTION_FULL_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
Expand Down Expand Up @@ -558,11 +614,11 @@ jobs:
mae_consumer_build,
mce_consumer_build,
datahub_upgrade_build,
datahub_ingestion_build,
datahub_ingestion_slim_build,
]
steps:
- name: Disk Check
run: df -h .
run: df -h . && docker images
- name: Check out the repo
uses: actions/checkout@v3
- name: Set up JDK 11
Expand All @@ -574,19 +630,13 @@ jobs:
with:
python-version: "3.7"
cache: "pip"
- name: Disk Check
run: df -h .
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Disk Check
run: df -h .
run: ./metadata-ingestion/scripts/install_deps.sh.
- name: Build datahub cli
run: |
./gradlew :metadata-ingestion:install
- name: Disk Check
run: df -h .
- name: Docker Image Check
run: docker images
run: df -h . && docker images
- name: Download GMS image
uses: ishworkh/docker-image-artifact-download@v1
if: ${{ needs.setup.outputs.publish != 'true' }}
Expand Down Expand Up @@ -632,6 +682,8 @@ jobs:
if: ${{ needs.setup.outputs.publish != 'true' }}
with:
image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Disk Check
run: df -h . && docker images
- name: run quickstart
env:
DATAHUB_TELEMETRY_ENABLED: false
Expand Down Expand Up @@ -661,6 +713,8 @@ jobs:
}'
- name: Remove Source Code
run: find ./*/* ! -path "./metadata-ingestion*" ! -path "./smoke-test*" ! -path "./gradle*" -delete
- name: Disk Check
run: df -h . && docker images
- name: Smoke test
env:
RUN_QUICKSTART: false
Expand All @@ -672,7 +726,7 @@ jobs:
echo "$DATAHUB_VERSION"
./smoke-test/smoke.sh
- name: Disk Check
run: df -h .
run: df -h . && docker images
- name: store logs
if: failure()
run: |
Expand Down
2 changes: 1 addition & 1 deletion smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ describe("run managed ingestion", () => {
cy.waitTextVisible(testName)

cy.contains(testName).parent().within(() => {
cy.contains("Succeeded", {timeout: 30000})
cy.contains("Succeeded", {timeout: 1200000})
cy.clickOptionWithTestId("delete-button");
})
cy.clickOptionWithText("Yes")
Expand Down

0 comments on commit 2dc6b78

Please sign in to comment.