Merge branch 'master' into master

.github/actions/ci-optimization/action.yml

@@ -1,5 +1,5 @@
-name: 'Identify CI Optimizations'
-description: 'Determine if code changes are specific to certain modules.'
+name: "Identify CI Optimizations"
+description: "Determine if code changes are specific to certain modules."
 
 outputs:
   frontend-only:
@@ -44,27 +44,26 @@ outputs:
 runs:
   using: "composite"
   steps:
-    - uses: dorny/paths-filter@v2
+    - uses: dorny/paths-filter@v3
       id: filter
       with:
+        token: "" # Empty token forces it to use raw git commands.
         filters: |
           frontend:
             - "datahub-frontend/**"
             - "datahub-web-react/**"
-            - "smoke-test/tests/cypress/**"
             - "docker/datahub-frontend/**"
           ingestion:
             - "metadata-ingestion-modules/**"
             - "metadata-ingestion/**"
             - "metadata-models/**"
-            - "smoke-test/**"
-            - "docker/datahub-ingestion**"
+            - "docker/datahub-ingestion-base/**"
+            - "docker/datahub-ingestion/**"
           ingestion-base:
             - "docker/datahub-ingestion-base/**"
           docker:
             - "docker/**"
           backend:
-            - ".github/**"
             - "metadata-models/**"
             - "datahub-upgrade/**"
             - "entity-registry/**"
@@ -78,7 +77,6 @@ runs:
             - "metadata-utils/**"
             - "metadata-operation-context/**"
             - "datahub-graphql-core/**"
-            - "smoke-test/**"
             - "docker/**"
           kafka-setup:
             - "docker/kafka-setup/**"

.github/actions/docker-custom-build-and-push/action.yml

@@ -26,36 +26,59 @@ inputs:
   build-args:
     description: "List of build-time variables. Same as docker/build-push-action"
     required: false
-  tags:
-    # e.g. latest,head,sha12345
-    description: "List of tags to use for the Docker image"
+  image_tag:
+    # e.g. pr12345 OR head OR v0.1.2.3
+    description: "Main tag to use for the Docker image"
     required: true
+  flavor:
+    description: "Image flavor (e.g., slim, full)"
+    required: false
   target:
     description: "Sets the target stage to build"
     required: false
+  depot-project:
+    # Setting this will use native arm64 docker builds instead of QEMU emulation.
+    # This speeds up builds by 2-3x.
+    description: "Depot project id"
+    required: false
+
 outputs:
   image_tag:
     description: "Docker image tags"
     value: ${{ steps.docker_meta.outputs.tags }}
-  # image_name: ${{ env.DATAHUB_GMS_IMAGE }}
 
 runs:
   using: "composite"
 
   steps:
     - name: Docker meta
       id: docker_meta
-      uses: crazy-max/ghaction-docker-meta@v1
+      uses: docker/metadata-action@v5
       with:
-        # list of Docker images to use as base name for tags
         images: ${{ inputs.images }}
-        # add git short SHA as Docker tag
-        tag-custom: ${{ inputs.tags }}
-        tag-custom-only: true
+        flavor: |
+          latest=false
+        tags: |
+          type=raw,value=${{ inputs.image_tag }}
+          type=raw,value=head,suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }},enable={{is_default_branch}}
+          type=sha,prefix=,format=short,suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+
+    - name: Single Tag
+      id: single_tag
+      shell: bash
+      run: |
+        IMAGES="""
+        ${{ inputs.images }}
+        """
+        TAGS="""
+        ${{ inputs.image_tag }}
+        """
+        echo "SINGLE_IMAGE=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> "$GITHUB_OUTPUT"
+        echo "SINGLE_TAG=$(echo $IMAGES | tr '\n' ' ' | awk -F' |,' '{ print $1 }'):$(echo $TAGS | tr '\n' ' ' | awk -F' |,' '{ print $1 }')" >> "$GITHUB_OUTPUT"
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -68,17 +91,11 @@ runs:
         target: ${{ inputs.target }}
         load: true
         push: false
-        cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
-        cache-to: type=inline
-    - name: Single Tag
-      if: ${{ inputs.publish != 'true' }}
-      shell: bash
-      run: |
-        TAGS="""
-        ${{ steps.docker_meta.outputs.tags }}
-        """
-        echo "SINGLE_TAG=$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
-      id: single_tag
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
     - name: Upload image locally for testing (if not publishing)
       uses: ishworkh/docker-image-artifact-upload@v1
       if: ${{ inputs.publish != 'true' }}
@@ -88,19 +105,42 @@ runs:
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
-      if: ${{ inputs.publish == 'true' }}
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-      if: ${{ inputs.publish == 'true' }}
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
+    - name: Setup Depot CLI
+      uses: depot/setup-action@v1
+      if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }}
     - name: Login to DockerHub
       uses: docker/login-action@v3
       if: ${{ inputs.publish == 'true' }}
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
+
+    # Depot variant.
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v5
-      if: ${{ inputs.publish == 'true' }}
+      uses: depot/build-push-action@v1
+      if: ${{ inputs.publish == 'true' && inputs.depot-project != '' }}
+      with:
+        project: ${{ inputs.depot-project }}
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        build-args: ${{ inputs.build-args }}
+        tags: ${{ steps.docker_meta.outputs.tags }}
+        target: ${{ inputs.target }}
+        push: true
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
+
+    - name: Build and Push Multi-Platform image
+      uses: docker/build-push-action@v6
+      if: ${{ inputs.publish == 'true' && inputs.depot-project == '' }}
       with:
         context: ${{ inputs.context }}
         file: ${{ inputs.file }}
@@ -109,7 +149,10 @@ runs:
         tags: ${{ steps.docker_meta.outputs.tags }}
         target: ${{ inputs.target }}
         push: true
-        cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
-        cache-to: type=inline
+        cache-from: |
+          type=registry,ref=${{ steps.single_tag.outputs.SINGLE_IMAGE }}:head${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=inline
 
     # TODO add code for vuln scanning?

.github/scripts/check_python_package.py

@@ -1,18 +1,33 @@
 import setuptools
+import os
 
 folders = ["./smoke-test/tests"]
 
 for folder in folders:
     print(f"Checking folder {folder}")
-    a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
-    b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
+    packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+    namespace_packages = [
+        i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i
+    ]
 
-    in_a_not_b = set(a) - set(b)
-    in_b_not_a = set(b) - set(a)
+    print("Packages found:", packages)
+    print("Namespace packages found:", namespace_packages)
+
+    in_packages_not_namespace = set(packages) - set(namespace_packages)
+    in_namespace_not_packages = set(namespace_packages) - set(packages)
+
+    if in_packages_not_namespace:
+        print(f"Packages not in namespace packages: {in_packages_not_namespace}")
+    if in_namespace_not_packages:
+        print(f"Namespace packages not in packages: {in_namespace_not_packages}")
+        for pkg in in_namespace_not_packages:
+            pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep))
+            print(f"Contents of {pkg_path}:")
+            print(os.listdir(pkg_path))
 
     assert (
-        len(in_a_not_b) == 0
-    ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
+        len(in_packages_not_namespace) == 0
+    ), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}"
     assert (
-        len(in_b_not_a) == 0
-    ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"
+        len(in_namespace_not_packages) == 0
+    ), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}"

.github/scripts/docker_helpers.sh

@@ -5,22 +5,22 @@ export MAIN_BRANCH="master"
 export MAIN_BRANCH_TAG="head"
 
 function get_short_sha {
-    echo $(git rev-parse --short "$GITHUB_SHA")
+    echo $(git rev-parse --short "$GITHUB_SHA"|head -c7)
 }
 
 export SHORT_SHA=$(get_short_sha)
 echo "SHORT_SHA: $SHORT_SHA"
 
 function get_tag {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g'),${SHORT_SHA}
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
 }
 
 function get_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
 }
 
 function get_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
 }
 
 function get_python_docker_release_v {
@@ -32,9 +32,17 @@ function get_unique_tag {
 }
 
 function get_unique_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
 }
 
 function get_unique_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+}
+
+function get_platforms_based_on_branch {
+    if [ "${{ github.event_name }}" == 'push' && "${{ github.ref }}" == "refs/heads/${MAIN_BRANCH}" ]; then
+        echo "linux/amd64,linux/arm64"
+    else
+        echo "linux/amd64"
+    fi
 }

.github/scripts/docker_logs.sh

@@ -0,0 +1,8 @@
+TARGET_DIR="${TARGET_DIR:=docker_logs}"
+TEST_STRATEGY="${TEST_STRATEGY:=}"
+
+mkdir -p "$TARGET_DIR"
+for name in `docker ps -a --format '{{.Names}}'`;
+do
+    docker logs "$name" >& "${TARGET_DIR}/${name}${TEST_STRATEGY}.log" || true
+done

.github/workflows/airflow-plugin.yml

@@ -52,18 +52,18 @@ jobs:
             extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.11"
-            extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt'
+            extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt'
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
       - uses: acryldata/sane-checkout-action@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"

.github/workflows/build-and-test.yml

@@ -57,18 +57,23 @@ jobs:
     timeout-minutes: 60
     needs: setup
     steps:
+      - name: Free up disk space
+        run: |
+          sudo apt-get remove 'dotnet-*' azure-cli || true
+          sudo rm -rf /usr/local/lib/android/ || true
+          sudo docker image prune -a -f || true
       - uses: szenius/[email protected]
         with:
           timezoneLinux: ${{ matrix.timezone }}
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:
           python-version: "3.10"
@@ -86,6 +91,10 @@ jobs:
             -x :metadata-ingestion-modules:airflow-plugin:check \
             -x :metadata-ingestion-modules:dagster-plugin:build \
             -x :metadata-ingestion-modules:dagster-plugin:check \
+            -x :metadata-ingestion-modules:prefect-plugin:build \
+            -x :metadata-ingestion-modules:prefect-plugin:check \
+            -x :metadata-ingestion-modules:gx-plugin:build \
+            -x :metadata-ingestion-modules:gx-plugin:check \
             -x :datahub-frontend:build \
             -x :datahub-web-react:build \
             --parallel
@@ -118,7 +127,7 @@ jobs:
     steps:
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Quickstart Compose Validation
@@ -131,4 +140,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: Event File
-          path: ${{ github.event_path }}
+          path: ${{ github.event_path }}

.github/workflows/check-datahub-jars.yml

@@ -29,12 +29,12 @@ jobs:
     steps:
       - uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: check ${{ matrix.command }} jar

.github/workflows/code-checks.yml

@@ -32,7 +32,7 @@ jobs:
     steps:
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: run check ${{ matrix.command }}