From 5d27e3393ebe729078d3b458a7b0ef4f2c56df84 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Aug 2023 16:04:30 -0500 Subject: [PATCH] split base/full/slim --- .github/workflows/docker-unified.yml | 144 ++++++++++++------ .../e2e/mutations/managed_ingestion.js | 2 +- 2 files changed, 100 insertions(+), 46 deletions(-) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 351c6bc2d9f5e..25454ed63f8d6 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -32,7 +32,7 @@ env: DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup" DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade" DATAHUB_INGESTION_BASE_IMAGE: "acryldata/datahub-ingestion-base" - DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion" + DATAHUB_INGESTION_FULL_IMAGE: "acryldata/datahub-ingestion" DATAHUB_INGESTION_SLIM_IMAGE: "acryldata/datahub-ingestion-slim" jobs: @@ -417,8 +417,8 @@ jobs: file: ./docker/elasticsearch-setup/Dockerfile platforms: linux/amd64,linux/arm64/v8 - datahub_ingestion_build: - name: Build and Push DataHub Ingestion Docker Images + datahub_ingestion_base_build: + name: Build and Push DataHub Ingestion (Base) Docker Image runs-on: ubuntu-latest outputs: tag: ${{ steps.tag.outputs.tag }} @@ -434,10 +434,6 @@ jobs: filters: | datahub-ingestion-base: - 'docker/datahub-ingestion-base/**' - datahub-ingestion-slim: - - 'docker/datahub-ingestion-slim/**' - datahub-ingestion: - - 'docker/datahub-ingestion/**' - name: Build and push Base Image if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }} uses: ./.github/actions/docker-custom-build-and-push @@ -451,6 +447,29 @@ jobs: context: . file: ./docker/datahub-ingestion-base/Dockerfile platforms: linux/amd64,linux/arm64/v8 + - name: Compute DataHub Ingestion (Base) Tag + id: tag + run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT + + datahub_ingestion_slim_build: + name: Build and Push DataHub Ingestion (Slim) Docker Images + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + needs: [setup, datahub_ingestion_base_build] + steps: + - name: Check out the repo + uses: actions/checkout@v3 + with: + fetch-depth: 800 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + datahub-ingestion-base: + - 'docker/datahub-ingestion-base/**' + datahub-ingestion-slim: + - 'docker/datahub-ingestion-slim/**' - name: Build schemas if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true' }} run: ./gradlew :metadata-events:mxe-schemas:build @@ -469,12 +488,69 @@ jobs: context: . file: ./docker/datahub-ingestion-slim/Dockerfile platforms: linux/amd64,linux/arm64/v8 + - name: Compute (Slim) Tag + id: tag + run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT + datahub_ingestion_slim_scan: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + name: "[Monitoring] Scan Datahub Ingestion Slim images for vulnerabilities" + runs-on: ubuntu-latest + needs: [setup, datahub_ingestion_slim_build] + steps: + - name: Checkout # adding checkout step just to make trivy upload happy + uses: actions/checkout@v3 + - name: Download image Slim Image + uses: ishworkh/docker-image-artifact-download@v1 + if: ${{ needs.setup.outputs.publish != 'true' }} + with: + image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} + - name: Run Trivy vulnerability scanner Slim Image + uses: aquasecurity/trivy-action@0.8.0 + env: + TRIVY_OFFLINE_SCAN: true + with: + image-ref: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "CRITICAL,HIGH" + ignore-unfixed: true + vuln-type: "os,library" + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + + datahub_ingestion_full_build: + name: Build and Push DataHub Ingestion (Full) Docker Images + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + needs: [setup, datahub_ingestion_slim_build] + steps: + - name: Check out the repo + uses: actions/checkout@v3 + with: + fetch-depth: 800 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + datahub-ingestion-base: + - 'docker/datahub-ingestion-base/**' + datahub-ingestion-slim: + - 'docker/datahub-ingestion-slim/**' + datahub-ingestion: + - 'docker/datahub-ingestion/**' - name: Build and push Full Image if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }} uses: ./.github/actions/docker-custom-build-and-push with: images: | - ${{ env.DATAHUB_INGESTION_IMAGE }} + ${{ env.DATAHUB_INGESTION_FULL_IMAGE }} build-args: | DOCKER_VERSION=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }} tags: ${{ needs.setup.outputs.tag }} @@ -484,18 +560,17 @@ jobs: context: . file: ./docker/datahub-ingestion/Dockerfile platforms: linux/amd64,linux/arm64/v8 - - name: Compute Tag + - name: Compute Tag (Full) id: tag - run: | - echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT - datahub_ingestion_scan: + run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion-slim == 'true') && needs.setup.outputs.tag || 'head' }}" >> $GITHUB_OUTPUT + datahub_ingestion_full_scan: permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities" runs-on: ubuntu-latest - needs: [setup, datahub_ingestion_build] + needs: [setup, datahub_ingestion_full_build] steps: - name: Checkout # adding checkout step just to make trivy upload happy uses: actions/checkout@v3 @@ -503,32 +578,13 @@ jobs: uses: ishworkh/docker-image-artifact-download@v1 if: ${{ needs.setup.outputs.publish != 'true' }} with: - image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }} + image: ${{ env.DATAHUB_INGESTION_FULL_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} - name: Run Trivy vulnerability scanner Full Image uses: aquasecurity/trivy-action@0.8.0 env: TRIVY_OFFLINE_SCAN: true with: - image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Docker Prune - run: docker image prune "${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }}" - - name: Download image Slim Image - uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }} - - name: Run Trivy vulnerability scanner Slim Image - uses: aquasecurity/trivy-action@0.8.0 - env: - TRIVY_OFFLINE_SCAN: true - with: - image-ref: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.datahub_ingestion_build.outputs.tag }} + image-ref: ${{ env.DATAHUB_INGESTION_FULL_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} format: "template" template: "@/contrib/sarif.tpl" output: "trivy-results.sarif" @@ -558,11 +614,11 @@ jobs: mae_consumer_build, mce_consumer_build, datahub_upgrade_build, - datahub_ingestion_build, + datahub_ingestion_slim_build, ] steps: - name: Disk Check - run: df -h . + run: df -h . && docker images - name: Check out the repo uses: actions/checkout@v3 - name: Set up JDK 11 @@ -574,19 +630,13 @@ jobs: with: python-version: "3.7" cache: "pip" - - name: Disk Check - run: df -h . - name: Install dependencies run: ./metadata-ingestion/scripts/install_deps.sh - - name: Disk Check - run: df -h . - name: Build datahub cli run: | ./gradlew :metadata-ingestion:install - name: Disk Check - run: df -h . - - name: Docker Image Check - run: docker images + run: df -h . && docker images - name: Download GMS image uses: ishworkh/docker-image-artifact-download@v1 if: ${{ needs.setup.outputs.publish != 'true' }} @@ -632,12 +682,14 @@ jobs: if: ${{ needs.setup.outputs.publish != 'true' }} with: image: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }}:${{ needs.setup.outputs.unique_tag }} + - name: Disk Check + run: df -h . && docker images - name: run quickstart env: DATAHUB_TELEMETRY_ENABLED: false DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }} DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_SLIM_IMAGE }} - ACTIONS_VERSION: ${{ needs.datahub_ingestion_build.outputs.tag }} + ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag }} ACTIONS_EXTRA_PACKAGES: 'acryl-datahub-actions[executor] acryl-datahub-actions' ACTIONS_CONFIG: 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml' run: | @@ -661,6 +713,8 @@ jobs: }' - name: Remove Source Code run: find ./*/* ! -path "./metadata-ingestion*" ! -path "./smoke-test*" ! -path "./gradle*" -delete + - name: Disk Check + run: df -h . && docker images - name: Smoke test env: RUN_QUICKSTART: false @@ -672,7 +726,7 @@ jobs: echo "$DATAHUB_VERSION" ./smoke-test/smoke.sh - name: Disk Check - run: df -h . + run: df -h . && docker images - name: store logs if: failure() run: | diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js index fd6703a7ef29e..de16ada8e83da 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js @@ -31,7 +31,7 @@ describe("run managed ingestion", () => { cy.waitTextVisible(testName) cy.contains(testName).parent().within(() => { - cy.contains("Succeeded", {timeout: 30000}) + cy.contains("Succeeded", {timeout: 1200000}) cy.clickOptionWithTestId("delete-button"); }) cy.clickOptionWithText("Yes")